Migration

import os
import hashlib
import sqlite3
import numpy as np
import logging

# Dateisystem-Crawler
def crawl_directory(root_dir):
    for dirpath, dirnames, filenames in os.walk(root_dir):
        for filename in filenames:
            file_path = os.path.join(dirpath, filename)
            size = os.path.getsize(file_path)
            created_at = os.path.getctime(file_path)
            hash_value = generate_hash(file_path)
            yield (file_path, size, created_at, hash_value)

# Hashwertgenerierung
def generate_hash(file_path):
    hash_alg = hashlib.md5()
    with open(file_path, 'rb') as f:
        while True:
            data = f.read(65536)
            if not data:
                break
            hash_alg.update(data)
    return hash_alg.hexdigest()

# SQLite-Datenbank
def create_db(db_file):
    conn = sqlite3.connect(db_file)
    c = conn.cursor()
    c.execute('''CREATE TABLE IF NOT EXISTS files
                 (id INTEGER PRIMARY KEY AUTOINCREMENT,
                  path TEXT UNIQUE,
                  size INTEGER,
                  created_at REAL,
                  hash_value TEXT)''')
    conn.commit()
    conn.close()

# Analyse auf doppelte Einträge
def find_duplicates(db_file, threshold=0.9):
    conn = sqlite3.connect(db_file)
    c = conn.cursor()
    c.execute('SELECT id, hash_value FROM files')
    rows = c.fetchall()
    hashes = np.array([np.frombuffer(bytes.fromhex(row[1]), dtype=np.uint8) for row in rows])
    similarities = np.dot(hashes, hashes.T) / (np.linalg.norm(hashes, axis=1) * np.linalg.norm(hashes, axis=1)[:, np.newaxis])
    np.fill_diagonal(similarities, 0)
    duplicates = np.argwhere(similarities > threshold)
    for pair in duplicates:
        file1 = rows[pair[0]]
        file2 = rows[pair[1]]
        logging.info(f'Duplicate found: {file1[0]} ({file1[1]}) and {file2[0]} ({file2[1]})')
    conn.close()

# Log-Erstellung
def setup_logging(log_file):
    logging.basicConfig(filename=log_file, level=logging.INFO,
                        format='%(asctime)s - %(levelname)s - %(message)s')

if __name__ == '__main__':
    root_dir = '/path/to/root/directory'
    db_file = 'file_info.db'
    log_file = 'migration.log'
    setup_logging(log_file)
    create_db(db_file)
    conn = sqlite3.connect(db_file)
    c = conn.cursor()
    c.execute('BEGIN TRANSACTION')
    for file_info in crawl_directory(root_dir):
        try:
            c.execute('INSERT INTO files (path, size, created_at, hash_value) VALUES (?, ?, ?, ?)', file_info)
        except sqlite3.IntegrityError:
            logging.warning(f'File {file_info[0]} already exists in database')
    c.execute('COMMIT')
    conn.close()
    find_duplicates(db_file)