Merge pull request #12 from CybercentreCanada/repo_update

Update project to follow template

.cruft.json

@@ -0,0 +1,23 @@
+{
+  "template": "[email protected]:CybercentreCanada/assemblyline-service-template.git",
+  "commit": "44fff28eb9f9537cd9564d16dd95a85e88b174be",
+  "checkout": null,
+  "context": {
+    "cookiecutter": {
+      "service_name": "urlcreator",
+      "__svc_name": "urlcreator",
+      "__repository": "assemblyline-service-urlcreator",
+      "__pkg_name": "urlcreator",
+      "class_name": "URLCreator",
+      "short_description": "This Assemblyline service creates URI files for URIs that were extracted by other services with an associated score that is high enough, or if the URI fits a certain set of criteria.",
+      "short_description_fr": "Ce service d'Assemblyline crée des fichiers URI pour les URIs qui sont extrait par d'autres services associés à un score suffisament élevé, ou si le URI rencontre certains critères.",
+      "stage": "POST",
+      "category": "Static Analysis",
+      "org_name_full": "CybercentreCanada",
+      "org_name_short": "cccs",
+      "license": "mit",
+      "_template": "[email protected]:CybercentreCanada/assemblyline-service-template.git"
+    }
+  },
+  "directory": null
+}

.dockerignore

@@ -0,0 +1,32 @@
+Dockerfile
+.idea
+.git
+.gitignore
+.vscode
+.dockerignore
+
+pipelines
+venv
+.venv
+env
+.env
+test
+tests
+examples
+docs
+
+build
+dist
+**/__pycache__
+**/*.pyc
+
+pip-log.txt
+pip-delete-this-directory.txt
+.tox
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.log

.gitignore

@@ -1,3 +1,7 @@
+# Created by https://www.toptal.com/developers/gitignore/api/python,vim,visualstudiocode
+# Edit at https://www.toptal.com/developers/gitignore?templates=python,vim,visualstudiocode
+
+### Python ###
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
@@ -6,11 +10,6 @@ __pycache__/
 # C extensions
 *.so
 
-# IDE files
-.pydevproject
-.python-version
-.idea
-
 # Distribution / packaging
 .Python
 build/
@@ -60,6 +59,22 @@ cover/
 *.mo
 *.pot
 
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
 # PyBuilder
 .pybuilder/
 target/
@@ -71,6 +86,43 @@ target/
 profile_default/
 ipython_config.py
 
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
 # Environments
 .env
 .venv
@@ -80,5 +132,85 @@ ENV/
 env.bak/
 venv.bak/
 
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
 # Cython debug symbols
 cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+### Python Patch ###
+# Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration
+poetry.toml
+
+# ruff
+.ruff_cache/
+
+# LSP config files
+pyrightconfig.json
+
+### Vim ###
+# Swap
+[._]*.s[a-v][a-z]
+!*.svg  # comment out if you don't need vector files
+[._]*.sw[a-p]
+[._]s[a-rt-v][a-z]
+[._]ss[a-gi-z]
+[._]sw[a-p]
+
+# Session
+Session.vim
+Sessionx.vim
+
+# Temporary
+.netrwhist
+*~
+# Auto-generated tag files
+tags
+# Persistent undo
+[._]*.un~
+
+### VisualStudioCode ###
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+!.vscode/*.code-snippets
+
+# Local History for Visual Studio Code
+.history/
+
+# Built Visual Studio Code Extensions
+*.vsix
+
+### VisualStudioCode Patch ###
+# Ignore all local history of files
+.history
+.ionide
+
+# End of https://www.toptal.com/developers/gitignore/api/python,vim,visualstudiocode

.vscode/launch.json

@@ -5,15 +5,15 @@
     "version": "0.2.0",
     "configurations": [
         {
-            "name": "RunServiceOnce",
+            "name": "RunServiceOnce URLCreator",
             "type": "python",
             "request": "launch",
             "module": "assemblyline_v4_service.dev.run_service_once",
             "cwd": "${workspaceFolder}",
             "args": [
                 "-d",
-                "urlcreator.URLCreator",
-                "/path/to/sample"
+                "urlcreator.urlcreator.URLCreator",
+                "${file}"
             ],
             "justMyCode": false,
         },
@@ -25,7 +25,7 @@
             "env": {
                 "SERVICE_MANIFEST_PATH": "service_manifest.yml",
                 "PRIVILEGED": "true",
-                "SERVICE_PATH": "urlcreator.URLCreator",
+                "SERVICE_PATH": "urlcreator.urlcreator.URLCreator",
                 "TASKING_DIR": "/tmp/URLCreator"
             },
             "console": "internalConsole",

CONTRIBUTING.md

@@ -0,0 +1,69 @@
+# Assemblyline contributing guide
+
+This guide covers the basics of how to contribute to the Assemblyline project.
+
+Python code should follow the PEP8 guidelines defined here:
+[PEP8 Guidelines](https://www.python.org/dev/peps/pep-0008/).
+
+## Tell us want you want to build/fix
+
+Before you start coding anything you should connect with the Assemblyline community via the
+[Assemblyline Discord server](https://discord.gg/GUAy9wErNu) and/or the
+[central Assemblyline GitHub project](https://github.com/CybercentreCanada/assemblyline/issues) to make sure no one
+else is working on the same thing and that whatever you are going to build still fits with the vision of the system.
+
+## Git workflow
+
+- Clone the repo to your own account
+- Checkout and pull the latest commits from the master branch
+- Make a branch
+- Work on your modifications and make sure your changes work as expected
+- When you're satisfied with your changes, create a pull requests to the Assemblyline repo
+
+#### Transfer your service repo
+
+If you've worked on a new service that you want to be included in the default service selection you'll have to transfer
+the associated repo into our control.
+
+#### You are not allow to merge:
+
+Even if you try to merge in your pull request, you will be denied. Only a few people in our team are allowed to merge
+code into our repositories.
+
+We check for new pull requests every day and will merge them in once they have been approved by someone in our team.
+
+# Guide de contribution d'Assemblyline
+
+Ce guide couvre les bases afin de contribuer au projet Assemblyline.
+
+Le code Python doit suivre les directives PEP8 définies ici:
+[Directives PEP8](https://www.python.org/dev/peps/pep-0008/).
+
+## Dites-nous que vous voulez construire / réparer
+
+Avant de commencer à coder quoi que ce soit, vous devriez vous connecter à la communauté Assemblyline via le
+[Serveur Discord Assemblyline](https://discord.gg/GUAy9wErNu) et/ou le
+[projet GitHub central Assemblyline](https://github.com/CybercentreCanada/assemblyline/issues) pour vous assurer que
+personne d'autre ne travaille sur la même chose et que tout ce que vous allez construire correspond toujours à la vision
+du système.
+
+## Flux de travail avec Git
+
+- Clonez le référentiel sur votre propre compte
+- Changez de branche pour la branche principale et la synchroniser avec le serveur de référence
+- Faire une nouvelle branche
+- Travaillez sur ce que vous souhaitez et assurez-vous que vos modifications fonctionnent comme prévu
+- Lorsque vous êtes satisfait de vos modifications, créez une demande de fusion sur le référentiel d'Assemblyline
+
+#### Transférer votre référentiel de service
+
+Si vous avez travaillé sur un nouveau service que vous souhaitez inclure dans la sélection de service par défaut, vous
+devrez transférer le référentiel associé sous notre contrôle.
+
+#### Vous n'êtes pas autorisé à compléter une fusion:
+
+Même si vous tentez de compléter une demande de fusion, vous serez refusé. Seules quelques personnes de notre équipe
+sont autorisées à fusionner dans nos référentiels.
+
+Nous vérifions les nouvelles demande de fusion tous les jours et les fusionnerons une fois qu'elles auront été approuvées
+par quelqu'un de notre équipe.

Dockerfile

@@ -1,21 +1,33 @@
 ARG branch=latest
 FROM cccs/assemblyline-v4-service-base:$branch
 
+# Python path to the service class from your service directory
 ENV SERVICE_PATH urlcreator.urlcreator.URLCreator
 
-# Install python dependencies
-COPY requirements.txt requirements.txt
-RUN pip install --no-cache-dir --user --requirement requirements.txt && rm -rf ~/.cache/pip
+# Install apt dependencies
+USER root
+COPY pkglist.txt /tmp/setup/
+RUN apt-get update && \
+    apt-get upgrade -y && \
+    apt-get install -y --no-install-recommends \
+    $(grep -vE "^\s*(#|$)" /tmp/setup/pkglist.txt | tr "\n" " ") && \
+    rm -rf /tmp/setup/pkglist.txt /var/lib/apt/lists/*
 
-# Switch to assemblyline user
+# Install python dependencies
 USER assemblyline
+COPY requirements.txt requirements.txt
+RUN pip install \
+    --no-cache-dir \
+    --user \
+    --requirement requirements.txt && \
+    rm -rf ~/.cache/pip
 
 # Copy service code
 WORKDIR /opt/al_service
 COPY . .
 
 # Patch version in manifest
-ARG version=4.0.0.dev1
+ARG version=1.0.0.dev1
 USER root
 RUN sed -i -e "s/\$SERVICE_TAG/$version/g" service_manifest.yml
 

LICENSE

@@ -1,11 +1,27 @@
 MIT License
 
-Copyright (c) 2020 Crown Copyright, Government of Canada (Canadian Centre for Cyber Security / Communications Security Establishment)
+Copyright (c) 2025 Crown Copyright, Government of Canada
+(Canadian Centre for Cyber Security / Communications Security Establishment)
 
-Copyright title to all 3rd party software distributed with Assemblyline (AL) is held by the respective copyright holders as noted in those files. Users are asked to read the 3rd Party Licenses referenced with those assets.
+Copyright title to all 3rd party software distributed with Assemblyline (AL)
+is held by the respective copyright holders as noted in those files. Users
+are asked to read the 3rd Party Licenses referenced with those assets.
 
-Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
 
-The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
 
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.