From b0c71c7b6dc0c0adb507d78f401e95e7ab0f5a38 Mon Sep 17 00:00:00 2001
From: KASR <karim.asrih@gmail.com>
Date: Wed, 3 May 2023 17:31:28 +0200
Subject: scripts : platform independent script to verify sha256 checksums
 (#1203)

* python script to verify the checksum of the llama models

Added Python script for verifying SHA256 checksums of files in a directory, which can run on multiple platforms. Improved the formatting of the output results for better readability.

* Update README.md

update to the readme for improved readability and to explain the usage of the python checksum verification script

* update the verification script

I've extended the script based on suggestions by @prusnak

The script now checks the available RAM, is there is enough to check the file at once it will do so. If not the file is read in chunks.

* minor improvment

small change so that the available ram is checked and not the total ram

* remove the part of the code that reads the file at once if enough ram is available

based on suggestions from @prusnak i removed the part of the code that checks whether the user had enough ram to read the entire model at once. the file is now always read in chunks.

* Update verify-checksum-models.py

quick fix to pass the git check
---
 scripts/verify-checksum-models.py | 78 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 78 insertions(+)
 create mode 100644 scripts/verify-checksum-models.py

(limited to 'scripts/verify-checksum-models.py')

diff --git a/scripts/verify-checksum-models.py b/scripts/verify-checksum-models.py
new file mode 100644
index 00000000..811372e4
--- /dev/null
+++ b/scripts/verify-checksum-models.py
@@ -0,0 +1,78 @@
+import os
+import hashlib
+
+def sha256sum(file):
+    block_size = 16 * 1024 * 1024  # 16 MB block size
+    b  = bytearray(block_size)
+    file_hash = hashlib.sha256()
+    mv = memoryview(b)
+    with open(file, 'rb', buffering=0) as f: 
+        while True:
+            n = f.readinto(mv)
+            if not n:
+                break
+            file_hash.update(mv[:n])
+        
+    return file_hash.hexdigest()
+
+# Define the path to the llama directory (parent folder of script directory)
+llama_path = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))
+
+# Define the file with the list of hashes and filenames
+hash_list_file = os.path.join(llama_path, "SHA256SUMS")
+
+# Check if the hash list file exists
+if not os.path.exists(hash_list_file):
+    print(f"Hash list file not found: {hash_list_file}")
+    exit(1)
+
+# Read the hash file content and split it into an array of lines
+with open(hash_list_file, "r") as f:
+    hash_list = f.read().splitlines()
+
+# Create an array to store the results
+results = []
+
+# Loop over each line in the hash list
+for line in hash_list:
+    # Split the line into hash and filename
+    hash_value, filename = line.split("  ")
+
+    # Get the full path of the file by joining the llama path and the filename
+    file_path = os.path.join(llama_path, filename)
+
+    # Informing user of the progress of the integrity check
+    print(f"Verifying the checksum of {file_path}")
+
+    # Check if the file exists
+    if os.path.exists(file_path):
+        # Calculate the SHA256 checksum of the file using hashlib
+        file_hash = sha256sum(file_path)
+
+        # Compare the file hash with the expected hash
+        if file_hash == hash_value:
+            valid_checksum = "V"
+            file_missing = ""
+        else:
+            valid_checksum = ""
+            file_missing = ""
+    else:
+        valid_checksum = ""
+        file_missing = "X"
+
+    # Add the results to the array
+    results.append({
+        "filename": filename,
+        "valid checksum": valid_checksum,
+        "file missing": file_missing
+    })
+
+
+# Print column headers for results table
+print("\n" + "filename".ljust(40) + "valid checksum".center(20) + "file missing".center(20))
+print("-" * 80)
+
+# Output the results as a table
+for r in results:
+    print(f"{r['filename']:40} {r['valid checksum']:^20} {r['file missing']:^20}")
+
-- 
cgit v1.2.3