v 0.0.15

Signed-off-by: Avishrant Sharma <avishrants@gmail.com>

Author: AvishrantsSh

CHANGELOG

@@ -1,7 +1,8 @@
 Change Log
 ==========
-0.0.13 (13/07/2021)
+0.0.15 (18/07/2021)
 ------------------
-- Errors are now part of scan_result and not raised (except Threshold Value Error)
-- Removed mulithreading and go routines
-- Add unittests
+- Add experimental parameter 'use_buffer' for large files.
+- Generate License mapping compaitble with Scancode.io
+- Update BabelStoneIDS license category
+- Add max_size parameter to stop scan

README.md

@@ -1,3 +1,71 @@
-# LicensePackage
-A Python module to find valid copyright and license expressions in a file.
-This module is based on Google LicenseClassifier.
+# Golicense-Classifier
+A Python based module to find valid copyright and license expressions in a file.
+
+_Note: This module is based on Google LicenseClassifier._
+
+## Installation
+Currently, this package only supports Linux Platform. Work is in progress for Windows and Mac.
+
+To install from Pypi, use
+```sh
+pip install golicense-classifier
+```
+
+## Usage
+To get started, import `LicenseClassifier` class from the module as
+
+```python
+from LicenseClassifier.classifier import LicenseClassifier
+```
+
+_Note: Work on Copyright Statement is still in progress. Expect some issues, mostly with binary files_
+
+The class comes bundled with several functions for scanning purpose.
+
+1. `scan_directory`
+        
+    This method is used to recursively walk through a directory and find license expressions and copyright statements. It returns a dictionary object with keys `header` and `files`.
+    
+    ### Usage
+    ___
+    ```python
+    classifier = LicenseClassifier()
+    res = classifier.scan_directory('PATH_TO_DIR')
+    ```
+    ### Optional Parameters
+    ___
+    - `max_size`
+        
+        Maximum size of file in MB. Default is set to 10MB. Set `max_size < 0` to ignore size constraints
+
+    - `use_buffer`
+        
+        `(Experimental)` Set `True` to use buffered file scanning. `max_size` will be used as buffer size.
+
+
+2. `scan_file`
+
+    This method is used to find license expressions and copyright statements on a single file.
+    
+    ### Usage
+    ___
+    ```python
+    classifier = LicenseClassifier()
+    res = classifier.scan_file('PATH_TO_FILE')
+    ```
+    ### Optional Parameters
+    ___
+    - `max_size`
+        
+        Maximum size of file in MB. Default is set to 10MB. Set `max_size < 0` to ignore size constraints
+
+    - `use_buffer`
+        
+        `(Experimental)` Set `True` to use buffered file scanning. `max_size` will be used as buffer size.
+
+## Setting Custom Scanning Threshold
+
+You can set custom threshold for scanning purpose that best suits your need. For this, you can use parameter `threshold` while making object as
+```python
+classifier = LicenseClassifier(threshold = 0.9)
+```

setup.py

@@ -5,10 +5,10 @@
 
 setuptools.setup(
     name="golicense_classifier",
-    version="0.0.14",
+    version="0.0.15",
     author="AvishrantSh (Avishrant Sharma)",
     author_email="<avishrants@gmail.com>",
-    description="A Python based License Classifier based on Google License Classifier",
+    description="A Python based License Classification and Copyright Statement Detection tool based on Google License Classifier",
     long_description=long_description,
     long_description_content_type="text/markdown",
     url="https://github.com/AvishrantsSh/LicensePackage",

src/LicenseClassifier/classifier.py

@@ -9,16 +9,16 @@ class LicenseClassifier:
     Base Class
     """
 
-    _ROOT = os.path.dirname(__file__)
+    __ROOT = os.path.dirname(__file__)
 
     # Shared Library
-    _so = ctypes.cdll.LoadLibrary(os.path.join(_ROOT, "compiled/libmatch.so"))
-    _init = _so.CreateClassifier
-    _init.argtypes = [ctypes.c_char_p, ctypes.c_double]
+    __so = ctypes.cdll.LoadLibrary(os.path.join(__ROOT, "compiled/libmatch.so"))
+    __init = __so.CreateClassifier
+    __init.argtypes = [ctypes.c_char_p, ctypes.c_double]
 
-    _scanfile = _so.ScanFile
-    _scanfile.argtypes = [ctypes.c_char_p, ctypes.c_int]
-    _scanfile.restype = ctypes.c_char_p
+    __scanfile = __so.ScanFile
+    __scanfile.argtypes = [ctypes.c_char_p, ctypes.c_int, ctypes.c_bool]
+    __scanfile.restype = ctypes.c_char_p
 
     def __init__(self, threshold: float = 0.8) -> None:
         """
@@ -32,24 +32,33 @@ def __init__(self, threshold: float = 0.8) -> None:
         if not 0 < threshold <= 1:
             raise ValueError("Threshold out of bounds (0 < threshold <= 1)")
 
-        self._init(
-            os.fsencode(os.path.join(LicenseClassifier._ROOT, "licenses/")), threshold
+        self.__init(
+            os.fsencode(os.path.join(LicenseClassifier.__ROOT, "licenses/")), threshold
         )
 
-    def scan_directory(self, location: str):
+    def scan_directory(self, location: str, max_size=10, use_buffer=False):
         """
         Function to find valid license and copyright expressions for files in `location`.
 
         Parameters
         ----------
         location : str
             Path to location of directory to scan.
+        max_size : int
+            Maximum size of file in MB. Default is set to 10MB. Set `max_size < 0` to ignore size constraints
+        use_buffer : bool
+            `(Experimental)` Set `True` to use buffered file scanning. `max_size` will be used as buffer size.
         """
         result = []
         start_time = datetime.now(timezone.utc)
         for (dirpath, _, filenames) in os.walk(location):
             result += [
-                self.scan_file(os.path.join(dirpath, file)) for file in filenames
+                self.scan_file(
+                    os.path.join(dirpath, file),
+                    max_size=max_size,
+                    use_buffer=use_buffer,
+                )
+                for file in filenames
             ]
 
         result = sorted(result, key=lambda k: k["path"])
@@ -64,15 +73,17 @@ def scan_directory(self, location: str):
                     "duration": (end_time - start_time).total_seconds(),
                     "files_count": len(result),
                     # ToDo: Add Error Expressions
-                    "errors": [],
+                    "errors": ['"' + location + '" does not exist']
+                    if not result
+                    else [],
                 }
             ],
             "files": result,
         }
 
         return scan_result
 
-    def scan_file(self, location: str, max_size=50):
+    def scan_file(self, location: str, max_size=10, use_buffer=False):
         """
         Function to find valid license and copyright expressions in `location`.
 
@@ -81,11 +92,15 @@ def scan_file(self, location: str, max_size=50):
         location : str
             Path to file.
         max_size : int
-            Maximum size of file in MB. Default is set to 50MB. Set `max_size < 0` to ignore size constraints
+            Maximum size of file in MB. Default is set to 10MB. Set `max_size < 0` to ignore size constraints
+        use_buffer : bool
+            `(Experimental)` Set `True` to use buffered file scanning. `max_size` will be used as buffer size. Recommended for large files.
         """
         # ToDo: DS Marshalling
 
-        json_string = os.fsdecode(self._scanfile(os.fsencode(location), max_size))
+        json_string = os.fsdecode(
+            self.__scanfile(os.fsencode(location), max_size, use_buffer)
+        )
 
         scan_result = json.loads(json_string)
 
@@ -141,7 +156,7 @@ def scan_file(self, location: str, max_size=50):
     "BSD-4-Clause": ["bsd-original", "Permissive"],
     "BSD-Protection": ["bsd-protection", "Copyleft"],
     "BSL-1.0": ["boost-1.0", "Permissive"],
-    "BabelstoneIDS": ["BabelstoneIDS", ""],
+    "BabelstoneIDS": ["BabelstoneIDS", "Public Domain"],
     "Beerware": ["beerware", "Permissive"],
     "BitTorrent-1.1": ["bittorrent-1.1", "Copyleft Limited"],
     "Business-Source-License-1.1": ["bsl-1.1", "Source-available"],

src/LicenseClassifier/compiled/libmatch.h

@@ -69,7 +69,7 @@ extern "C" {
 #endif
 
 extern void CreateClassifier(char* license, GoFloat64 defaultThreshold);
-extern char* ScanFile(char* fpaths, GoInt maxSize);
+extern char* ScanFile(char* fpaths, GoInt maxSize, GoUint8 useBuffer);
 
 #ifdef __cplusplus
 }