Reworking identification

- PARAM.SFO reader (sfo.py)
- Search for param first then use title there to identify
- long_description (setup.py)
- Multiman styling by default
- Don't check crc32 as it is very slow

.gitignore

@@ -1,18 +1,22 @@
 *.iso
 *.ird
 *.gz
+*.db
 
 PS3_GAME/
 PS3_UPDATE/
 PS3_DISC.SFB
 
+*/PS3_GAME
+*/PS3_UPDATE
+*PS3_DISC.SFB
+
 tools/keys/*
 tools/*/*
 tools/*.db
 tools/*.dat
 
-libra/data/*.db
-*.db
+libray/data/*.db
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

CHANGELOG.md

@@ -3,6 +3,18 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
+## [0.0.7] - Unreleased
+### Added
+- PARAM.SFO reader (sfo.py)
+- Now searches for PARAM.SFO first and uses game title from there instead of crc32.
+- long_description in setup.py for description on PyPI
+
+### Changed
+- Multiman styling by default if PARAM.SFO is found
+
+### Removed
+- No longer checks crc32, see Added. Might re-add as last fallback later.
+
 ## [0.0.6] - 2021-06-05
 ### Fixed
 - Issue #6: fix decrypting using disc key not working

README.md

@@ -132,14 +132,19 @@ If you get any other errors, or have any other problem with libray, please [crea
 
 ## Development
 
-[see also](http://www.psdevwiki.com/ps3/Bluray_disc#Encryption) ([archive.fo](https://archive.fo/hN1E6))
+[Bluray disc encryption](http://www.psdevwiki.com/ps3/Bluray_disc#Encryption) ([archive.fo](https://archive.fo/hN1E6))
+
+[.SFO](https://www.psdevwiki.com/ps3/PARAM.SFO) ([archive.fo](https://archive.fo/HLJZG))
+
+[TITLE_ID for Physical Media](https://www.psdevwiki.com/ps3/Template:TITLE_ID_for_Physical_Media) ([archive.fo](https://archive.fo/R8tCz))
 
 [7bit encoded int / RLE / CLP](https://github.com/microsoft/referencesource/blob/1acafe20a789a55daa17aac6bb47d1b0ec04519f/mscorlib/system/io/binaryreader.cs#L582-L600)
 
 clp = compressed length prefix
 
-## Building and Deployment
+## Deployment
 
+0. `pip3 install wheel twine`
 1. Place redump keys in tools/keys and .dat in tools/
 2. Run keys2db.py, ensure it made a file in libray/data/keys.db
 3. Run `python3 setup.py sdist bdist_wheel`

libray/core.py

@@ -162,6 +162,42 @@ def crc32(filename):
     return "%08X" % (crc32 & 0xFFFFFFFF)
 
 
+def serial_country(title):
+  """Get country from disc serial / productcode / title_id"""
+
+  if title[2] == 'A':
+    return 'Asia'
+  if title[2] == 'C':
+    return 'China'
+  if title[2] == 'E':
+    return 'Europe'
+  if title[2] == 'H':
+    return 'Hong Kong'
+  if title[2] == 'J' or title[2] == 'P':
+    return 'Japan'
+  if title[2] == 'K':
+    return 'Korea'
+  if title[2] == 'U':
+    return 'USA'
+
+  raise ValueError('Unknown country?!')
+
+
+def multiman_title(title):
+  """Fix special characters in title for Multiman style"""
+
+  replace = {
+    ':': ' -',
+    '/': '-',
+    '™': ' -',
+  }
+
+  for key, val in replace.items():
+    title = title.replace(key, val)
+
+  return title
+
+
 # Main functions
 
 
@@ -173,7 +209,7 @@ def decrypt(args):
 
   input_iso = iso.ISO(args)
 
-  input_iso.decrypt(args)
+  input_iso.decrypt(args) # TODO: some of the logic should probably be moved up here instead of residing in the decrypt function
 
 
 def encrypt(args):

libray/iso.py

@@ -21,6 +21,7 @@
 
 import sys
 import sqlite3
+import pathlib
 import pkg_resources
 from tqdm import tqdm
 from Crypto.Cipher import AES
@@ -29,9 +30,11 @@ from Crypto.Cipher import AES
 try:
   from libray import core
   from libray import ird
+  from libray import sfo
 except ImportError:
   import core
   import ird
+  import sfo
 
 
 class ISO:
@@ -93,11 +96,54 @@ class ISO:
 
       self.regions = self.read_regions(input_iso, args.iso)
 
-      # Seek to the start of region 2, '+ 16' skips a section containing some 'playstation'
+      # Seek to the start of sector 2, '+ 16' skips a section containing some 'playstation'
       input_iso.seek(core.SECTOR + 16)
 
       self.game_id = input_iso.read(16).decode('utf8').strip()
 
+      # Find PARAM.SFO
+
+      core.vprint('Searching for PARAM.SFO', args)
+
+      input_iso.seek(0)
+      counter = 1
+      found_param = False
+
+      while True:
+
+        data = input_iso.read(4)
+
+        if not data:
+          break
+
+        if data == b'\x00\x50\x53\x46':
+          found_param = True
+          break
+
+        input_iso.seek((core.SECTOR * counter))
+
+        counter += 1
+
+      game_title = ''
+
+      if found_param:
+        input_iso.seek(input_iso.tell() - 4)
+        try:
+          param = sfo.SFO(input_iso)
+          core.vprint('PARAM.SFO found', args)
+          game_title = core.multiman_title(param['TITLE'])
+
+          if args.verbose and not args.quiet:
+            param.print_info()
+
+          # Set output to multiman style
+
+          if not args.output:
+            args.output = '%s [%s].iso' % (game_title, param['TITLE_ID'])
+
+        except:
+          core.warning('Failed reading SFO')
+
     cipher = AES.new(core.ISO_SECRET, AES.MODE_CBC, core.ISO_IV)
 
     if not args.decryption_key:
@@ -109,20 +155,35 @@ class ISO:
         core.vprint('Checking for bundled redump keys', args)
 
         try:
-          db = sqlite3.connect(pkg_resources.resource_filename(__name__, 'data/keys.db'))
+          try:
+            db = sqlite3.connect(pkg_resources.resource_filename(__name__, 'data/keys.db'))
+          except FileNotFoundError:
+            db = sqlite3.connect((pathlib.Path(__file__).resolve() / 'data/') / 'keys.db')
           c = db.cursor()
 
-          core.vprint('Calculating crc32', args)
+          if not game_title:
+            raise ValueError
 
-          crc32 = core.crc32(args.iso)
+          core.vprint('Trying to find redump key based on size and game title', args)
 
-          keys = c.execute('SELECT * FROM games WHERE crc32=?', [crc32.lower()]).fetchall()
+          #core.vprint('Calculating crc32', args)
+
+          #input_iso.seek(0)
+
+          # crc32 = core.crc32(args.iso)
+
+          #keys = c.execute('SELECT * FROM games WHERE crc32=?', [crc32.lower()]).fetchall()
+
+          keys = c.execute('SELECT * FROM games WHERE lower(name) LIKE ? AND size = ?', ['%' + game_title.lower() + '%', str(self.size)]).fetchall()
 
           if keys:
 
             self.disc_key = keys[0][-1]
 
-            core.vprint('.ISO identified as "%s"' % keys[0][0], args)
+            if not self.disc_key:
+              raise ValueError
+
+            core.vprint('Found potential redump key: "%s"' % keys[0][0], args)
 
             redump = True
 
@@ -159,6 +220,7 @@ class ISO:
     """Decrypt self using args from argparse."""
 
     core.vprint('Decrypting with disc key: %s' % self.disc_key.hex(), args)
+    core.vprint('Decrypted .iso is output to: %s' % args.output, args)
 
     with open(args.iso, 'rb') as input_iso:
 

libray/sfo.py

@@ -0,0 +1,124 @@
+# -*- coding: utf8 -*-
+
+# libray - Libre Blu-Ray PS3 ISO Tool
+# Copyright © 2018 -2021 Nichlas Severinsen
+#
+# This file is part of libray.
+#
+# libray is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# libray is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with libray.  If not, see <https://www.gnu.org/licenses/>.
+
+
+try:
+  from libray import core
+except ImportError:
+  import core
+
+
+class SFO:
+  """Class for handling .sfo files
+
+  Attributes:
+    magic: Magic header
+    version: .SFO version
+    key_table_start: Absolute offset for key_table in .SFO
+    data_table_start: Absolute offset for index_table in .SFO
+    tables_entries: Number of entries in index_table and key_table
+    key_data: Parsed keys and data tables from .SFO transformed into dict
+  """
+
+  def __init__(self, fp):
+
+    self.file_start = fp.tell()
+
+    # Header
+
+    self.magic = fp.read(4)
+    self.version = fp.read(4)
+    self.key_table_start = core.to_int(fp.read(4), 'little')
+    self.data_table_start = core.to_int(fp.read(4), 'little')
+    self.tables_entries = core.to_int(fp.read(4), 'little')
+
+    # Index table
+
+    index_table = []
+
+    for _ in range(0, self.tables_entries):
+      index_table.append({
+        'key_offset': core.to_int(fp.read(2), 'little'),
+        'data_fmt': fp.read(2),
+        'data_len': core.to_int(fp.read(4), 'little'),
+        'data_max_len': core.to_int(fp.read(4), 'little'),
+        'data_offset': core.to_int(fp.read(4), 'little'),
+      })
+
+    # Key table
+
+    key_table = []
+
+    for i in range(0, self.tables_entries):
+
+      # Seek to absolute offset + relative offset of key
+
+      fp.seek(self.file_start + self.key_table_start + index_table[i]['key_offset'])
+
+      # Read key string until nullbyte
+
+      key = ''
+
+      while True:
+
+        data = fp.read(1)
+
+        if data == b'\x00':
+          break
+
+        key += data.decode('utf8')
+
+      key_table.append(key)
+
+    # Data table
+
+    self.key_data = {}
+
+    for i in range(0, self.tables_entries):
+
+      # Seek to absolute offset + relative offset of data
+
+      fp.seek(self.file_start + self.data_table_start + index_table[i]['data_offset'])
+
+      if index_table[i]['data_fmt'] == b'\x04\x02': #UTF8
+        data = fp.read(index_table[i]['data_len'] - 1).decode('utf8')
+      elif index_table[i]['data_fmt'] == b'\x04\x04': #int32
+        data = core.to_int(fp.read(index_table[i]['data_len']), 'little')
+      else: # Meh
+        data = fp.read(index_table[i]['data_len'])
+
+      self.key_data[key_table[i]] = data
+
+
+  def __getitem__(self, key):
+    """Overload [] so we can directly select data using key from .SFO"""
+    return self.key_data[key]
+
+
+  def print_info(self):
+
+    print('Magic:', self.magic)
+    print('Version: ', self.version)
+    print('key_table_start:', self.key_table_start)
+    print('data_table_start:', self.data_table_start)
+    print('tables_entries:', self.tables_entries)
+
+    print(self.key_data)
+

setup.py

@@ -1,12 +1,20 @@
 #!/usr/bin/env python3
 # -*- coding: utf8 -*-
 
+
 from setuptools import setup
 
+
+with open('README.md') as f:
+  long_description = f.read()
+
+
 setup(
   name="libray",
-  version="0.0.6",
+  version="0.0.7",
   description='A Libre (FLOSS) Python application for unencrypting, extracting, repackaging, and encrypting PS3 ISOs',
+  long_description=long_description,
+  long_description_content_type='text/markdown',
   author="Nichlas Severinsen",
   author_email="ns@nsz.no",
   url="https://notabug.org/necklace/libray",

tools/keys2db.py

@@ -55,6 +55,8 @@ if __name__ == '__main__':
 
     datfile = any_dats[0]
 
+    warnings = 0
+
     with open(datfile, 'r') as infile:
 
         soup = bs4.BeautifulSoup(infile.read(), features='html5lib')
@@ -70,7 +72,7 @@ if __name__ == '__main__':
                 with open(cwd / ('keys/' + name + '.key'), 'rb') as keyfile:
                     entry.append(keyfile.read())
             except FileNotFoundError:
-                print('Warning: key not found for ' + name)
+                warnings += 1
                 c.execute('INSERT INTO games (name, size, crc32, md5, sha1) VALUES (?, ?, ?, ?, ?)', entry)
                 continue
 
@@ -82,6 +84,7 @@ if __name__ == '__main__':
 
     shutil.copyfile(db_path, ((cwd.parent / 'libray') / 'data/') / db_path.name)
 
+    print('Warning: no keyfiles for %s titles' % str(warnings))