diff --git a/.gitignore b/.gitignore
index 68e8e94..90a3288 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,6 +6,14 @@ PS3_GAME/
PS3_UPDATE/
PS3_DISC.SFB
+tools/keys/*
+tools/*/*
+tools/*.db
+tools/*.dat
+
+libra/data/*.db
+*.db
+
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
diff --git a/CHANGELOG.md b/CHANGELOG.md
index a13d520..69a39d6 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -3,13 +3,16 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
-## [0.0.6] - 2020-06-02
+## [0.0.6] - 2021-06-05
### Fixed
-
- Issue #6: fix decrypting using disc key not working
### Added
- Added .iso re-encryption with -r / --re-encrypt, default output is game_id_e.iso (example: BLUS-0000_e.iso)
+- Added ability to bundle redump keys in the package data, libray now checks if it already has the key for the .iso
+
+### Changed
+- Changed printing to use `[*]` in front
## [0.0.5] - 2020-08-03
### Fixed
diff --git a/README.md b/README.md
index 698520d..db8e5ff 100644
--- a/README.md
+++ b/README.md
@@ -66,13 +66,13 @@ There's a compiled list of compatible drives here: [https://rpcs3.net/quickstart
### 1. Decrypt
-On some systems (eg. Linux), you can decrypt directly from the disc.
+On some systems (eg. Linux), you can decrypt directly from the disc:
```
libray -i /dev/sr0 -o ps3_game_decrypted.iso
```
-Libray will automatically try to download an IRD decryption file for your iso. If you don't have internet connection, but you do have an .ird file you can specify that:
+Libray is bundled with redump keys and will automatically try to decrypt the .iso if it finds a compatible key. If not, it will try to download an IRD decryption file for your iso. If you don't have internet connection, but you do have an .ird file you can specify that:
```
libray -i /dev/sr0 -k game_ird_file.ird -o ps3_game_decrypted.iso
@@ -138,11 +138,19 @@ If you get any other errors, or have any other problem with libray, please [crea
clp = compressed length prefix
+## Building and Deployment
+
+1. Place redump keys in tools/keys and .dat in tools/
+2. Run keys2db.py, ensure it made a file in libray/data/keys.db
+3. Run `python3 setup.py sdist bdist_wheel`
+4. Run `twine upload dist/*`
+
## Todo
- Extract ISO (currently doable with `7z x output.iso`)
-- Repackage (unextract) and reencrypt iso?
+- Repackage (unextract) iso
- Test .irds with version < 9
- Custom command to backup all irds available
- Unit tests
-
+- Download .irds from vimm.net?
+- Parallelization?
diff --git a/libray/core.py b/libray/core.py
index 7a3d6b6..b30e73c 100644
--- a/libray/core.py
+++ b/libray/core.py
@@ -22,6 +22,7 @@
import os
import sys
import stat
+import zlib
import shutil
import requests
from bs4 import BeautifulSoup
@@ -89,13 +90,22 @@ def read_seven_bit_encoded_int(fileobj, order):
def error(msg):
"""Print fatal error message and terminate"""
- print('ERROR: %s' % msg)
+ print('[ERROR] %s' % msg)
sys.exit(1)
-def warning(msg):
- """Print a warning message"""
- print('WARNING: %s. Continuing regardless.' % msg)
+def warning(msg, args):
+ """Print a warning message. Warning messages can be silenced with --quiet"""
+
+ if not args.quiet:
+ print('[WARNING] %s. Continuing regardless.' % msg)
+
+
+def vprint(msg, args):
+ """Vprint, verbose print, can be silenced with --quiet"""
+
+ if not args.quiet:
+ print('[*] ' + msg)
def download_ird(ird_name):
@@ -136,6 +146,22 @@ def ird_by_game_id(game_id):
return(ird_name)
+def crc32(filename):
+ """Calculate crc32 for file"""
+
+ with open(filename, 'rb') as infile:
+
+ crc32 = 0
+
+ while True:
+ data = infile.read(65536)
+ if not data:
+ break
+ crc32 = zlib.crc32(data, crc32)
+
+ return "%08X" % (crc32 & 0xFFFFFFFF)
+
+
# Main functions
diff --git a/libray/iso.py b/libray/iso.py
index 6c5ac25..a243bf2 100644
--- a/libray/iso.py
+++ b/libray/iso.py
@@ -20,6 +20,8 @@
import sys
+import sqlite3
+import pkg_resources
from tqdm import tqdm
from Crypto.Cipher import AES
@@ -96,35 +98,67 @@ class ISO:
self.game_id = input_iso.read(16).decode('utf8').strip()
- if args.verbose and not args.quiet:
- self.print_info()
-
cipher = AES.new(core.ISO_SECRET, AES.MODE_CBC, core.ISO_IV)
if not args.decryption_key:
if not args.ird:
- if not args.quiet:
- core.warning('No IRD file specified, finding required file')
- args.ird = core.ird_by_game_id(self.game_id) # Download ird
+ # No key or .ird specified. Let's first check if keys.db is packaged with this release
- self.ird = ird.IRD(args)
+ redump = False
- if self.ird.region_count != len(self.regions)-1:
- core.error('Corrupt ISO or error in IRD. Expected %s regions, found %s regions' % (self.ird.region_count, len(self.regions)-1))
+ core.vprint('Checking for bundled redump keys', args)
- if self.regions[-1]['start'] > self.size:
- core.error('Corrupt ISO or error in IRD. Expected filesize larger than %.2f GiB, actual size is %.2f GiB' % (self.regions[-1]['start'] / 1024**3, self.size / 1024**3 ) )
+ try:
+ db = sqlite3.connect(pkg_resources.resource_filename(__name__, 'data/keys.db'))
+ c = db.cursor()
- self.disc_key = cipher.encrypt(self.ird.data1)
+ core.vprint('Calculating crc32', args)
+
+ crc32 = core.crc32(args.iso)
+
+ keys = c.execute('SELECT * FROM games WHERE crc32=?', [crc32.lower()]).fetchall()
+
+ if keys:
+
+ self.disc_key = keys[0][-1]
+
+ core.vprint('.ISO identified as "%s"' % keys[0][0], args)
+
+ redump = True
+
+ else:
+ raise ValueError
+
+ except:
+ core.vprint('No keys found', args)
+
+ if not redump:
+
+ # Fallback to checking if an .ird exists
+
+ core.warning('No IRD file specified, finding required file', args)
+ args.ird = core.ird_by_game_id(self.game_id) # Download ird
+
+ self.ird = ird.IRD(args)
+
+ if self.ird.region_count != len(self.regions)-1:
+ core.error('Corrupt ISO or error in IRD. Expected %s regions, found %s regions' % (self.ird.region_count, len(self.regions)-1))
+
+ if self.regions[-1]['start'] > self.size:
+ core.error('Corrupt ISO or error in IRD. Expected filesize larger than %.2f GiB, actual size is %.2f GiB' % (self.regions[-1]['start'] / 1024**3, self.size / 1024**3 ) )
+
+ self.disc_key = cipher.encrypt(self.ird.data1)
else:
self.disc_key = core.to_bytes(args.decryption_key)
+ if args.verbose and not args.quiet:
+ self.print_info()
+
def decrypt(self, args):
"""Decrypt self using args from argparse."""
- if not args.quiet:
- print('Decrypting with disc key: %s' % self.disc_key.hex())
+ core.vprint('Decrypting with disc key: %s' % self.disc_key.hex(), args)
with open(args.iso, 'rb') as input_iso:
@@ -145,8 +179,8 @@ class ISO:
if not region['enc']:
while input_iso.tell() < region['end']:
data = input_iso.read(core.SECTOR)
- if not data and not args.quiet:
- core.warning('Trying to read past the end of the file')
+ if not data:
+ core.warning('Trying to read past the end of the file', args)
break
output_iso.write(data)
@@ -163,8 +197,8 @@ class ISO:
num >>= 8
data = input_iso.read(core.SECTOR)
- if not data and not args.quiet:
- core.warning('Trying to read past the end of the file')
+ if not data:
+ core.warning('Trying to read past the end of the file', args)
break
cipher = AES.new(self.disc_key, AES.MODE_CBC, bytes(iv))
@@ -177,14 +211,14 @@ class ISO:
if not args.quiet:
pbar.close()
- print('Decryption complete.')
+
+ core.vprint('Decryption complete!', args)
def encrypt(self, args):
"""Encrypt self using args from argparse."""
- if not args.quiet:
- print('Re-encrypting with disc key: %s' % self.disc_key.hex())
+ core.vprint('Re-encrypting with disc key: %s' % self.disc_key.hex(), args)
with open(args.iso, 'rb') as input_iso:
@@ -205,8 +239,8 @@ class ISO:
if not region['enc']:
while input_iso.tell() < region['end']:
data = input_iso.read(core.SECTOR)
- if not data and not args.quiet:
- core.warning('Trying to read past the end of the file')
+ if not data:
+ core.warning('Trying to read past the end of the file', args)
break
output_iso.write(data)
@@ -223,8 +257,8 @@ class ISO:
num >>= 8
data = input_iso.read(core.SECTOR)
- if not data and not args.quiet:
- core.warning('Trying to read past the end of the file')
+ if not data:
+ core.warning('Trying to read past the end of the file', args)
break
cipher = AES.new(self.disc_key, AES.MODE_CBC, bytes(iv))
@@ -235,6 +269,11 @@ class ISO:
if not args.quiet:
pbar.update(1)
+ if not args.quiet:
+ pbar.close()
+
+ core.vprint('Re-encryption complete!', args)
+
def print_info(self):
# TODO: This could probably have been a __str__? Who cares?
diff --git a/setup.py b/setup.py
index a6bd53a..c45d887 100755
--- a/setup.py
+++ b/setup.py
@@ -18,4 +18,6 @@ setup(
'requests==2.22.0',
'beautifulsoup4==4.7.1',
],
+ include_package_data=True,
+ package_data={'': ['data/keys.db']},
)
diff --git a/tools/keys2db.py b/tools/keys2db.py
new file mode 100755
index 0000000..197340d
--- /dev/null
+++ b/tools/keys2db.py
@@ -0,0 +1,89 @@
+#!/usr/bin/env python3
+# -*- coding: utf8 -*-
+
+# libray - Libre Blu-Ray PS3 ISO Tool
+# Copyright © 2018 - 2021 Nichlas Severinsen
+#
+# This file is part of libray.
+#
+# libray is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# libray is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with libray. If not, see .
+
+# This script transforms Datfile.dat and keys/*.key keyfiles into a sqlite3 keys.db
+# Keys.db is then moved to libray/data/keys.db and packaged with libray in setup.py.
+# Libray checks if this file is bundled with it and checks if it has a key for the .iso using a crc32 of it.
+# TODO: In theory we could add the game-serials (BLUS-0000) and check that first.
+
+
+import bs4
+import sys
+import shutil
+import sqlite3
+import pathlib
+
+
+if __name__ == '__main__':
+
+ db_path = pathlib.Path('keys.db')
+
+ if db_path.exists():
+ db_path.unlink()
+
+ db = sqlite3.connect(db_path)
+ c = db.cursor()
+
+ c.execute('CREATE TABLE games (name TEXT, size TEXT, crc32 TEXT, md5 TEXT, sha1 TEXT, key BLOB)')
+ db.commit()
+
+ cwd = pathlib.Path(__file__).resolve().parent
+
+ any_dats = [x for x in cwd.glob('*.dat')]
+
+ if not any_dats:
+ print('Error: No .dat file. Place the .dat file in the tools/ folder')
+ sys.exit()
+
+ datfile = any_dats[0]
+
+ with open(datfile, 'r') as infile:
+
+ soup = bs4.BeautifulSoup(infile.read(), features='html5lib')
+
+ for game in soup.find_all('game'):
+
+ name = game.find('description').text.strip()
+ attrs = game.find('rom').attrs
+
+ entry = [name, attrs['size'], attrs['crc'], attrs['md5'], attrs['sha1']]
+
+ try:
+ with open(cwd / ('keys/' + name + '.key'), 'rb') as keyfile:
+ entry.append(keyfile.read())
+ except FileNotFoundError:
+ print('Warning: key not found for ' + name)
+ c.execute('INSERT INTO games (name, size, crc32, md5, sha1) VALUES (?, ?, ?, ?, ?)', entry)
+ continue
+
+ c.execute('INSERT INTO games VALUES (?, ?, ?, ?, ?, ?)', entry)
+
+ db.commit()
+
+ db.close()
+
+ shutil.copyfile(db_path, ((cwd.parent / 'libray') / 'data/') / db_path.name)
+
+
+
+
+
+
diff --git a/tools/rpcs3.py b/tools/rpcs3.py
new file mode 100755
index 0000000..45bb146
--- /dev/null
+++ b/tools/rpcs3.py
@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+# -*- coding: utf8 -*-
+
+# libray - Libre Blu-Ray PS3 ISO Tool
+# Copyright © 2018 - 2021 Nichlas Severinsen
+#
+# This file is part of libray.
+#
+# libray is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# libray is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with libray. If not, see .
+
+# This is a script to find the redump names and game serial id's using rpcs3's compatibility list.
+# It puts the name, serial id, and some other info into an sqlite3 database.
+# That database can then be used to harcode serial id to keys into keys.db.
+# This script is not included in the release of libray.
+
+
+import bs4
+import string
+import sqlite3
+import pathlib
+import requests
+
+
+if __name__ == '__main__':
+
+ db_path = pathlib.Path('games.db')
+
+ #if db_path.exists():
+ # db_path.unlink()
+
+ db = sqlite3.connect(db_path)
+ c = db.cursor()
+
+ c.execute('CREATE TABLE IF NOT EXISTS games (serial TEXT PRIMARY KEY, country TEXT, type TEXT, name TEXT, redump_name TEXT)')
+ db.commit()
+
+ # "#" section
+
+ for i in range(1, 18):
+
+ url = 'https://rpcs3.net/compatibility?r=200&p=' + str(i)
+
+ print('Requesting page ' + str(i))
+
+ response = requests.get(url)
+
+ soup = bs4.BeautifulSoup(response.text, features='html5lib')
+
+ for row in soup.find_all('label', attrs={'class': 'compat-table-row'}):
+
+ columns = [column for column in row.find_all('div', attrs={'class': 'compat-table-cell'})]
+
+ for serial in columns[0].find_all(['img']):
+
+ game_id = serial.attrs['title'].strip()
+ country = serial.attrs['src'].split('/')[-1].split('.')[0]
+ game_type = columns[1].find('a').attrs['title'].strip()
+ name = columns[1].text.strip()
+
+ redump_name = ''
+
+ entry = [
+ game_id,
+ country,
+ game_type,
+ name,
+ country,
+ game_type,
+ name
+ ]
+
+ c.execute('INSERT INTO games VALUES (?, ?, ?, ?, ?) ON CONFLICT DO UPDATE SET country = ?, type = ?, name = ? ', entry)
+
+ db.commit()
+
+
+
+
+