Compare commits

..

No commits in common. "57ab326604ff33bf2d52515528b3fa2f4c866ec5" and "564da5df77cec0f55cc8546e8b1a2d46ca9b1e73" have entirely different histories.

4 changed files with 60 additions and 359 deletions

View File

@ -1,31 +0,0 @@
name: Backup to Gitea
on:
push:
branches:
- main # 你可以根据需要修改分支名称
jobs:
backup:
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v2
with:
fetch-depth: 0 # Fetch all history for all branches and tags
- name: Set up Git
run: |
git config --global user.name "biggerfish"
git config --global user.email "yuzhiyongcn@qq.com"
- name: Add Gitea remote
run: |
REPO_NAME=$(basename ${{ github.repository }})
git remote add gitea https://yu:${{secrets.GITEA_TOKEN}}@git.zhiyong.tech/yu/$REPO_NAME.git
- name: Push to Gitea
run: |
git push -u gitea --all --force
git push -u gitea --tags --force

View File

@ -1,5 +1,5 @@
# 照片分类器-python3
1. 对指定目录及子目录下的照片进行分类, 先按 年/月 分目录, 文件名重命名为"年-月-时间戳"
2. 每个照片文件的md5校验码存储到oracle新加坡云的mysql数据库'photo_classifier', 表名'photo', 用于检查照片是否重复, 重复的会跳过
3. 可以处理视频, 照片, 非照片的图片
4. 整理后的照片存放到"总仓库-照片视频", 被备份到多个云盘
2. 每个照片文件的md5校验码存储到oracle新加坡云的mysql数据库'photo_classifier', 用于检查照片是否重复, 重复的会跳过
3. 如果照片经过修改, 丢失EXIF信息, 将被跳过
4. 整理后的照片存放到"自动备份", 被备份到多个云盘

View File

@ -1,247 +0,0 @@
"""
根据读取的照片信息分类照片
分类:
目录名:2020\01
文件名:2020-01-时间戳
"""
import os
import sys
from posixpath import abspath
import exifread
import time
import shutil
import hashlib
import pymysql
import datetime
import pytz
from win32com.propsys import propsys, pscon
# 使用韩国oracle 数据库
class Classifier:
mode = "prod" # 开发模式(dev)还是产品模式(prod)
IMAGE_EXTENTIONS = ["jpg", "jpeg", "bmp", "png", "tif", "gif", "heic"]
VIDEO_EXTENTIONS = ["mp4", "avi", "rmvb", "mkv", "mov", "amr", "mpg"]
TEST_TABLE = "TEST_PHOTO"
TABLE = "PHOTO"
PHOTO_NO_DATE_KEYS = ["EXIF ExifVersion"]
PHOTO_DATE_KEYS = ["Image DateTime", "EXIF DateTimeOriginal"]
PHOTO_EXIF_KEYS = PHOTO_NO_DATE_KEYS + PHOTO_DATE_KEYS
SKIP_FOLDERS = ["System Volume Information", "$RECYCLE.BIN", ".stfolder"]
def __init__(self, input_folder, photo_output, video_output, image_output):
self.input = input_folder
self.photo_output = photo_output
self.video_output = video_output
self.image_output = image_output
self.processed_count = 0
self.table = self.TEST_TABLE if self.mode == "dev" else self.TABLE
pass
def connect_database(self):
self.db = pymysql.connect(
host="panel.zhiyong.tech",
user="yu_biggerfish",
password="jRHTbQrdkfNNTztH",
database="photo_classifier",
)
def close_database(self):
self.db.close()
def create_table(self):
self.connect_database()
cursor = self.db.cursor()
sql = "DROP TABLE IF EXISTS {}".format(self.table)
cursor.execute(sql)
print("删除表 {}".format(self.table))
sql = """CREATE TABLE {} (
ID INT NOT NULL AUTO_INCREMENT ,
MD5 VARCHAR(255) NOT NULL ,
PRIMARY KEY (ID), UNIQUE (MD5))
ENGINE = InnoDB;""".format(
self.table
)
cursor.execute(sql)
print("创建表 {}".format(self.table))
self.close_database()
def start(self):
self.connect_database()
self.process_folder(self.input)
self.delete_folders(self.input)
self.close_database()
def get_file_count(self, folder):
count = 0
for _, _, _files in os.walk(folder):
count += len(_files)
return count
def delete_folders(self, folder):
for root, dirs, files in os.walk(folder):
for dir in dirs:
if dir in self.SKIP_FOLDERS:
continue
abs_path = os.path.join(root, dir)
if os.path.isdir(abs_path):
if self.get_file_count(abs_path) == 0:
shutil.rmtree(abs_path)
print("删除目录: {}".format(abs_path))
def is_photo(self, file_name):
return self.is_image(file_name) and self.contains_exif(file_name)
def is_video(self, file_name):
for ext in self.VIDEO_EXTENTIONS:
if file_name.lower().endswith(ext):
return True
return False
def is_image(self, file_name):
for ext in self.IMAGE_EXTENTIONS:
if file_name.lower().endswith(ext):
return True
return False
def contains_exif(self, file_name):
with open(file_name, "rb") as reader:
tags = exifread.process_file(reader)
keys = [key for key in self.PHOTO_EXIF_KEYS if key in tags]
return len(keys) > 0
def process_folder(self, folder):
for root, dirs, files in os.walk(folder):
for file in files:
self.process_file(root, file)
def get_md5(self, file):
with open(file, "rb") as reader:
return hashlib.md5(reader.read()).hexdigest()
def process_file(self, root, file):
file_path = os.path.join(root, file)
if self.is_image(file_path) or self.is_video(file_path):
md5 = self.get_md5(file_path)
try:
self.validate(file_path, md5)
year, month, day = self.read_date(file_path)
new_name = self.rename_move(file_path, year, month, day, md5)
self.add_record(md5)
self.processed_count += 1
print(
"已处理 {}: {} --> {}".format(self.processed_count, file, new_name)
)
except Exception as e:
print(str(e))
else:
print("非图片或视频, 忽略文件: {}".format(file_path))
def add_record(self, md5):
try:
cursor = self.db.cursor()
sql = "INSERT INTO {}(MD5) VALUES('{}')".format(self.table, md5)
cursor.execute(sql)
self.db.commit()
except Exception as e:
print("插入记录 {} 到数据库photo_classifier失败: {}".format(md5, str(e)))
self.db.rollback()
raise e
def validate(self, file_path, md5):
# check if the md5 of the photo exists in database
try:
cursor = self.db.cursor()
sql = "SELECT MD5 FROM {} WHERE MD5='{}'".format(self.table, md5)
cursor.execute(sql)
record = cursor.fetchone()
if str(record) != "None":
os.remove(file_path)
raise Exception("重复文件 {} --> 删除".format(file_path))
except Exception as e:
raise e
if (not self.is_image(file_path)) and (not self.is_video(file_path)):
raise Exception("非图片或视频: {} --> 跳过".format(file_path))
def get_photo_create_date(self, file):
with open(file, "rb") as reader:
tags = exifread.process_file(reader)
keys = [key for key in self.PHOTO_DATE_KEYS if key in tags]
if len(keys) > 0:
key = keys[0]
origin_date = tags[key]
time_str = str(origin_date)
_date = time_str[:10].split(":")
year = _date[0]
month = _date[1]
day = _date[2]
return (year, month, day)
return None
def get_video_create_date(self, file):
try:
properties = propsys.SHGetPropertyStoreFromParsingName(file)
dt = properties.GetValue(pscon.PKEY_Media_DateEncoded).GetValue()
time_str = str(dt.astimezone(pytz.timezone("Asia/Shanghai")))
_date = time_str[:10].split("-")
year = _date[0]
month = _date[1]
day = _date[2]
return (year, month, day)
except:
return None
def read_date(self, file):
file = file.replace("/", "\\")
date = None
if self.is_photo(file):
date = self.get_photo_create_date(file) # 照片可能没有EXIF日期
elif self.is_video(file):
date = self.get_video_create_date(file) # 视频可能没有媒体创建日期
if not date: # 获取文件上次修改日期
time_str = os.path.getmtime(file)
time_str = str(datetime.datetime.fromtimestamp(time_str))
_date = time_str[:10].split("-")
year = _date[0]
month = _date[1]
day = _date[2]
date = (year, month, day)
return date
def rename_move(self, file_path, year, month, day, md5):
if self.is_image(file_path):
if self.is_photo(file_path):
output = self.photo_output
else:
output = self.image_output
elif self.is_video(file_path):
output = self.video_output
else:
raise Exception("移动文件失败, 非图片或视频: {}".format(file_path))
new_path = os.path.join(output, year, month, day)
if not os.path.exists(new_path):
os.makedirs(new_path)
file_name, file_ext = os.path.splitext(file_path)
new_name = year + "-" + month + "-" + day + "-" + md5 + file_ext
shutil.move(file_path, os.path.join(new_path, new_name))
return new_name
cf = Classifier(
input_folder="D:/待分类照片视频",
# input_folder="D:/总仓库-照片视频-bak",
photo_output="D:/总仓库-照片视频/总照片备份",
video_output="D:/总仓库-照片视频/总视频备份",
image_output="D:/总仓库-照片视频/总图片备份",
)
cf.start()
# cf.create_table()

View File

@ -1,12 +1,13 @@
"""
'''
根据读取的照片信息分类照片
分类:
目录名:2020\01
文件名:2020-01-时间戳
"""
处理过的文件名存为json文件: processed_files.json
如果照片经过修改, 丢失原始EXIF信息, 将会被跳过
'''
import os
import sys
from posixpath import abspath
import exifread
import time
@ -18,16 +19,15 @@ import pytz
from win32com.propsys import propsys, pscon
class Classifier1:
mode = "prod" # 开发模式(dev)还是产品模式(prod)
IMAGE_EXTENTIONS = ["jpg", "jpeg", "bmp", "png", "tif", "gif", "heic"]
VIDEO_EXTENTIONS = ["mp4", "avi", "rmvb", "mkv", "mov", "amr", "mpg"]
TEST_TABLE = "TEST_PHOTO"
TABLE = "PHOTO"
PHOTO_NO_DATE_KEYS = ["EXIF ExifVersion"]
PHOTO_DATE_KEYS = ["Image DateTime", "EXIF DateTimeOriginal"]
class Classifier():
mode = 'prod' # 开发模式(dev)还是产品模式(prod)
IMAGE_EXTENTIONS = ['jpg', 'jpeg', 'bmp', 'png']
VIDEO_EXTENTIONS = ['mp4']
TEST_TABLE = 'TEST_PHOTO'
TABLE = 'PHOTO'
PHOTO_NO_DATE_KEYS = ['EXIF ExifVersion']
PHOTO_DATE_KEYS = ['Image DateTime', 'EXIF DateTimeOriginal']
PHOTO_EXIF_KEYS = PHOTO_NO_DATE_KEYS + PHOTO_DATE_KEYS
SKIP_FOLDERS = ["System Volume Information", "$RECYCLE.BIN", ".stfolder"]
def __init__(self, input_folder, photo_output, video_output, image_output):
self.input = input_folder
@ -35,17 +35,11 @@ class Classifier1:
self.video_output = video_output
self.image_output = image_output
self.processed_count = 0
self.table = self.TEST_TABLE if self.mode == "dev" else self.TABLE
self.table = self.TEST_TABLE if self.mode == 'dev' else self.TABLE
pass
def connect_database(self):
self.db = pymysql.connect(
host="northflyfish.myqnapcloud.cn",
user="admin",
password="zhiyong@214",
database="photo_classifier",
port=3307,
)
self.db = pymysql.connect(host='bt.biggerfish.tech', user='admin', password='zhiyong214', database='photo_classifier')
def close_database(self):
self.db.close()
@ -54,19 +48,17 @@ class Classifier1:
self.connect_database()
cursor = self.db.cursor()
sql = "DROP TABLE IF EXISTS {}".format(self.table)
sql = 'DROP TABLE IF EXISTS {}'.format(self.table)
cursor.execute(sql)
print("删除表 {}".format(self.table))
print('删除表 {}'.format(self.table))
sql = """CREATE TABLE {} (
sql = '''CREATE TABLE {} (
ID INT NOT NULL AUTO_INCREMENT ,
MD5 VARCHAR(255) NOT NULL ,
PRIMARY KEY (ID), UNIQUE (MD5))
ENGINE = InnoDB;""".format(
self.table
)
ENGINE = InnoDB;'''.format(self.table)
cursor.execute(sql)
print("创建表 {}".format(self.table))
print('创建表 {}'.format(self.table))
self.close_database()
@ -78,20 +70,18 @@ class Classifier1:
def get_file_count(self, folder):
count = 0
for _, _, _files in os.walk(folder):
for (_, _, _files) in os.walk(folder):
count += len(_files)
return count
def delete_folders(self, folder):
for root, dirs, files in os.walk(folder):
for (root, dirs, files) in os.walk(folder):
for dir in dirs:
if dir in self.SKIP_FOLDERS:
continue
abs_path = os.path.join(root, dir)
if os.path.isdir(abs_path):
if self.get_file_count(abs_path) == 0:
shutil.rmtree(abs_path)
print("删除目录: {}".format(abs_path))
print('删除目录: {}'.format(abs_path))
def is_photo(self, file_name):
return self.is_image(file_name) and self.contains_exif(file_name)
@ -109,37 +99,32 @@ class Classifier1:
return False
def contains_exif(self, file_name):
with open(file_name, "rb") as reader:
with open(file_name, 'rb') as reader:
tags = exifread.process_file(reader)
keys = [key for key in self.PHOTO_EXIF_KEYS if key in tags]
return len(keys) > 0
def process_folder(self, folder):
for root, dirs, files in os.walk(folder):
for (root, dirs, files) in os.walk(folder):
for file in files:
self.process_file(root, file)
def get_md5(self, file):
with open(file, "rb") as reader:
with open(file, 'rb') as reader:
return hashlib.md5(reader.read()).hexdigest()
def process_file(self, root, file):
file_path = os.path.join(root, file)
if self.is_image(file_path) or self.is_video(file_path):
md5 = self.get_md5(file_path)
try:
self.validate(file_path, md5)
year, month, day = self.read_date(file_path)
new_name = self.rename_move(file_path, year, month, day, md5)
year, month = self.read_date(file_path)
new_name = self.rename_move(file_path, year, month)
self.add_record(md5)
self.processed_count += 1
print(
"已处理 {}: {} --> {}".format(self.processed_count, file, new_name)
)
print('已处理 {}: {} --> {}'.format(self.processed_count, file, new_name))
except Exception as e:
print(str(e))
else:
print("非图片或视频, 忽略文件: {}".format(file_path))
def add_record(self, md5):
try:
@ -148,7 +133,7 @@ class Classifier1:
cursor.execute(sql)
self.db.commit()
except Exception as e:
print("插入记录 {} 到数据库photo_classifier失败: {}".format(md5, str(e)))
print('插入记录 {} 到数据库photo_classifier失败: {}'.format(md5, str(e)))
self.db.rollback()
raise e
@ -159,45 +144,43 @@ class Classifier1:
sql = "SELECT MD5 FROM {} WHERE MD5='{}'".format(self.table, md5)
cursor.execute(sql)
record = cursor.fetchone()
if str(record) != "None":
if str(record) != 'None':
os.remove(file_path)
raise Exception("重复文件 {} --> 删除".format(file_path))
raise Exception('重复照片 {} --> 删除'.format(file_path))
except Exception as e:
raise e
if (not self.is_image(file_path)) and (not self.is_video(file_path)):
raise Exception("非图片或视频: {} --> 跳过".format(file_path))
raise Exception('非图片或视频: {} --> 跳过'.format(file_path))
def get_photo_create_date(self, file):
with open(file, "rb") as reader:
with open(file, 'rb') as reader:
tags = exifread.process_file(reader)
keys = [key for key in self.PHOTO_DATE_KEYS if key in tags]
if len(keys) > 0:
key = keys[0]
origin_date = tags[key]
time_str = str(origin_date)
_date = time_str[:10].split(":")
_date = time_str[:7].split(':')
year = _date[0]
month = _date[1]
day = _date[2]
return (year, month, day)
return (year, month)
return None
def get_video_create_date(self, file):
try:
properties = propsys.SHGetPropertyStoreFromParsingName(file)
dt = properties.GetValue(pscon.PKEY_Media_DateEncoded).GetValue()
time_str = str(dt.astimezone(pytz.timezone("Asia/Shanghai")))
_date = time_str[:10].split("-")
time_str = str(dt.astimezone(pytz.timezone('Asia/Shanghai')))
_date = time_str[:7].split('-')
year = _date[0]
month = _date[1]
day = _date[2]
return (year, month, day)
return (year, month)
except:
return None
def read_date(self, file):
file = file.replace("/", "\\")
file = file.replace('/', '\\')
date = None
if self.is_photo(file):
date = self.get_photo_create_date(file) # 照片可能没有EXIF日期
@ -207,14 +190,13 @@ class Classifier1:
if not date: # 获取文件上次修改日期
time_str = os.path.getmtime(file)
time_str = str(datetime.datetime.fromtimestamp(time_str))
_date = time_str[:10].split("-")
_date = time_str[:7].split('-')
year = _date[0]
month = _date[1]
day = _date[2]
date = (year, month, day)
date = (year, month)
return date
def rename_move(self, file_path, year, month, day, md5):
def rename_move(self, file_path, year, month):
if self.is_image(file_path):
if self.is_photo(file_path):
output = self.photo_output
@ -223,24 +205,21 @@ class Classifier1:
elif self.is_video(file_path):
output = self.video_output
else:
raise Exception("移动文件失败, 非图片或视频: {}".format(file_path))
raise Exception('移动文件失败, 非图片或视频: {}'.format(file_path))
new_path = os.path.join(output, year, month, day)
new_path = os.path.join(output, year, month)
if not os.path.exists(new_path):
os.makedirs(new_path)
file_name, file_ext = os.path.splitext(file_path)
new_name = year + "-" + month + "-" + day + "-" + md5 + file_ext
new_name = year + '-' + month + '-' + str(time.time()) + file_ext
shutil.move(file_path, os.path.join(new_path, new_name))
return new_name
cf = Classifier1(
input_folder="D:/待分类照片视频1",
# input_folder='z:/待分类照片视频/Picture',
photo_output="D:/总仓库-照片视频1/总照片备份",
video_output="D:/总仓库-照片视频1/总视频备份",
image_output="D:/总仓库-照片视频1/总图片备份",
)
cf = Classifier('D:/temp/相册',
photo_output='D:/总仓库-照片视频/总照片备份',
video_output='D:/总仓库-照片视频/总视频备份',
image_output='D:/总仓库-照片视频/总图片备份')
cf.start()
# cf.create_table()
cf.start()