sbomnix: release v1.4.5

- Introduce own nix files for each demo tool under scripts/. This change allows clearly stating dependencies for each tool. This change is also necessary in case we later decide to move some of the tools now under `scripts/` directory to their own repositories. - From now on, the default.nix in the root of this repository is only for `sbomnix` and `nixgraph` which are the main tools currently maintained in this repository. Other tools under `scripts/` can still be used via the flakes.nix or the shell.nix. - Add flake output targets for `repology_cli` and `nix_outdated` apps. - Introduce basic tests for `repology_cli` and `nix_outdated`. - Get rid of the `use_scm_version=True` in setup.py and read the version number from VERSION file instead. With this change, we can also remove the postPatch hack from default.nix. - Remove travis.yml as it's no longer used. - Update nix flake lock file. - Bump sbomnix version to v1.4.5. Signed-off-by: Henri Rosten <[email protected]>
tiiuae · Apr 6, 2023 · 0cf311a · 0cf311a
1 parent 1946007
commit 0cf311a
Show file tree

Hide file tree

Showing 22 changed files with 385 additions and 95 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,10 +1,9 @@
-# SPDX-FileCopyrightText: 2022 Technology Innovation Institute (TII)
+# SPDX-FileCopyrightText: 2023 Technology Innovation Institute (TII)
 #
 # SPDX-License-Identifier: Apache-2.0
 
 temp/
 venv/
-cpedict/
 build/
 *.egg-info/
 .eggs/
@@ -13,7 +12,7 @@ sbomnix_test_data/
 result
 *.py[cod]
 *.sqlite
-/*.csv
+*.csv
 /*.log
 /*.json
 /*.png

diff --git a/.travis.yml b/.travis.yml
diff --git a/VERSION b/VERSION
@@ -0,0 +1 @@
+1.4.5
diff --git a/VERSION.license b/VERSION.license
@@ -0,0 +1,3 @@
+# SPDX-FileCopyrightText: 2023 Technology Innovation Institute (TII)
+#
+# SPDX-License-Identifier: Apache-2.0
diff --git a/default.nix b/default.nix
@@ -4,36 +4,28 @@
 {
   pkgs ? import <nixpkgs> {},
   pythonPackages ? pkgs.python3Packages,
-  vulnix ? import ./scripts/vulnxscan/vulnix.nix { nixpkgs=pkgs.path; pkgs=pkgs; },
 }:
 
 pythonPackages.buildPythonPackage rec {
   pname = "sbomnix";
-  version = "1.4.4";
+  version = pkgs.lib.removeSuffix "\n" (builtins.readFile ./VERSION);
   format = "setuptools";
 
   src = ./.;
-
-  postPatch = ''
-    substituteInPlace setup.py \
-      --replace "use_scm_version=True," "version='${version}'," \
-      --replace "setup_requires=['setuptools_scm']," "setup_requires=[],"
-  '';
+  updateCpedict = import ./scripts/cpedict/update-cpedict.nix { pkgs=pkgs; };
+  makeWrapperArgs = [
+    "--prefix PATH : ${pkgs.lib.makeBinPath [ pkgs.nix pkgs.graphviz updateCpedict ]}"
+  ];
 
   propagatedBuildInputs = [ 
     pkgs.reuse
-    pkgs.grype
-    pkgs.curl
-    vulnix
-    pythonPackages.numpy
-    pythonPackages.pandas
     pythonPackages.colorlog
-    pythonPackages.tabulate
-    pythonPackages.wheel
+    pythonPackages.graphviz
+    pythonPackages.numpy
     pythonPackages.packageurl-python
+    pythonPackages.pandas
     pythonPackages.requests
-    pythonPackages.graphviz
+    pythonPackages.tabulate
   ];
-
   pythonImportsCheck = [ "sbomnix" ];
 }
diff --git a/flake.lock b/flake.lock
diff --git a/flake.nix b/flake.nix
@@ -9,12 +9,18 @@
   outputs = { self, nixpkgs }:
     let
       pkgs = import nixpkgs { system = "x86_64-linux"; };
+      vulnxscan = import ./scripts/vulnxscan/vulnxscan.nix { pkgs = pkgs; };
+      repology_cli = import ./scripts/repology/repology_cli.nix { pkgs = pkgs; };
+      nix_outdated = import ./scripts/nixupdate/nix_outdated.nix { pkgs = pkgs; };
       sbomnix = import ./default.nix { pkgs = pkgs; };
       sbomnix-shell = import ./shell.nix { pkgs = pkgs; };
     in rec {
 
       # nix package
       packages.x86_64-linux = {
+        inherit repology_cli;
+        inherit nix_outdated;
+        inherit vulnxscan;
         inherit sbomnix;
         default = sbomnix;
       };
@@ -34,7 +40,19 @@
       # nix run .#vulnxscan
       apps.x86_64-linux.vulnxscan = {
         type = "app";
-        program = "${self.packages.x86_64-linux.sbomnix}/bin/vulnxscan";
+        program = "${self.packages.x86_64-linux.vulnxscan}/bin/vulnxscan.py";
+      };
+
+      # nix run .#repology_cli
+      apps.x86_64-linux.repology_cli = {
+        type = "app";
+        program = "${self.packages.x86_64-linux.repology_cli}/bin/repology_cli.py";
+      };
+
+      # nix run .#nix_outdated
+      apps.x86_64-linux.nix_outdated= {
+        type = "app";
+        program = "${self.packages.x86_64-linux.nix_outdated}/bin/nix_outdated.py";
       };
 
       # nix develop

diff --git a/requirements.txt b/requirements.txt
@@ -3,11 +3,10 @@
 # SPDX-License-Identifier: Apache-2.0
 
 # common
-pandas
 colorlog
-tabulate
-wheel
+pandas
 reuse
+tabulate
 
 # sbomnix
 packageurl-python
@@ -19,14 +18,14 @@ graphviz
 requests
 
 # repology
-requests-ratelimiter
-requests-cache
 bs4
 packaging
+requests-ratelimiter
+requests-cache
 
 # dev requirements
+black
+jsonschema
 pycodestyle
 pylint
-black
 pytest
-jsonschema
diff --git a/scripts/cpedict/update-cpedict.nix b/scripts/cpedict/update-cpedict.nix
@@ -0,0 +1,35 @@
+# SPDX-FileCopyrightText: 2023 Technology Innovation Institute (TII)
+#
+# SPDX-License-Identifier: Apache-2.0
+{
+  pkgs ? import <nixpkgs> {},
+}:
+
+pkgs.stdenv.mkDerivation rec {
+  doCheck = true;
+  pname = "update-cpedict.sh";
+  version = pkgs.lib.removeSuffix "\n" (builtins.readFile ../../VERSION);
+  src = ./update-cpedict.sh;
+
+  path = pkgs.lib.makeBinPath ([
+    pkgs.coreutils
+    pkgs.curl
+    pkgs.gnugrep
+    pkgs.gnused
+    pkgs.gzip
+  ]);
+
+  checkInputs = [ pkgs.shellcheck ];
+  buildInputs = [ pkgs.bash ];
+  unpackPhase = ''
+    cp ${src} ${pname}
+  '';
+  checkPhase = ''
+    shellcheck ${pname}
+  '';
+  installPhase = ''
+    mkdir -p $out/bin
+    cp ${pname} $out/bin/${pname}
+    chmod +x $out/bin/${pname}
+  '';
+}
diff --git a/scripts/update-cpedict.sh → scripts/cpedict/update-cpedict.sh b/scripts/update-cpedict.sh → scripts/cpedict/update-cpedict.sh
diff --git a/scripts/nixupdate/README.md b/scripts/nixupdate/README.md
@@ -4,9 +4,24 @@ SPDX-FileCopyrightText: 2023 Technology Innovation Institute (TII)
 SPDX-License-Identifier: Apache-2.0
 -->
 
-# nix_outdated.py
+# nix_outdated
 
-`nix_outdated.py` is a command line tool to list outdated nix dependencies for given target nix out path. By default, the script outputs runtime dependencies for the given nix out path that appear outdated in nixpkgs 'nix_unstable' channel - the list of output packages would potentially need a PR to update the package in nixpkgs to the latest upstream release version specified in the output table column 'version_upstream'. The list of output packages is in priority order based on how many other packages depend on the potentially outdated package.
+`nix_outdated` is a command line tool to list outdated nix dependencies for given target nix out path. By default, the script outputs runtime dependencies for the given nix out path that appear outdated in nixpkgs 'nix_unstable' channel - the list of output packages would potentially need a PR to update the package in nixpkgs to the latest upstream release version specified in the output table column 'version_upstream'. The list of output packages is in priority order based on how many other packages depend on the potentially outdated package.
+
+## Running as Nix Flake
+`nix_outdated` can be run as a [Nix flake](https://nixos.wiki/wiki/Flakes) from the `tiiuae/sbomnix` repository:
+```bash
+# '--' signifies the end of argument list for `nix`.
+# '--help' is the first argument to `nix_outdated`
+$ nix run github:tiiuae/sbomnix#nix_outdated -- --help
+```
+
+or from a local repository:
+```bash
+$ git clone https://github.com/tiiuae/sbomnix
+$ cd sbomnix
+$ nix run .#nix_outdated -- --help
+```
 
 ## Running from Nix Development Shell
 

diff --git a/scripts/nixupdate/nix-visualize.nix b/scripts/nixupdate/nix-visualize.nix
@@ -15,8 +15,8 @@ pythonPackages.buildPythonPackage rec {
   src = pkgs.fetchFromGitHub {
     owner = "henrirosten";
     repo = pname;
-    rev = "5b451b5ab7a199ecdd94a2c1fa5aa92120a89941";
-    sha256 = "sha256-wRhgq5aljOa1uyIpaJSH7XgIrBgOm9LdHnSUKlholdU=";
+    rev = "7fedf92eacffd5c42c50f7fec72f61a377c9ccf3";
+    hash = "sha256-Z1r8XHszoUnQinl63yXvQG6Czp5HnYNG37AY+EEiT4w=";
   };
   propagatedBuildInputs = with pythonPackages; [
     matplotlib

diff --git a/scripts/nixupdate/nix_outdated.nix b/scripts/nixupdate/nix_outdated.nix
@@ -0,0 +1,31 @@
+# SPDX-FileCopyrightText: 2023 Technology Innovation Institute (TII)
+#
+# SPDX-License-Identifier: Apache-2.0
+{
+  pkgs ? import <nixpkgs> {},
+  pythonPackages ? pkgs.python3Packages,
+}:
+
+pythonPackages.buildPythonPackage rec {
+  pname = "nix_outdated";
+  version = pkgs.lib.removeSuffix "\n" (builtins.readFile ../../VERSION);
+  format = "setuptools";
+
+  src = ../../.;
+  sbomnix = import ../../default.nix { pkgs=pkgs; };
+  repology_cli = import ../repology/repology_cli.nix { pkgs=pkgs; };
+  nix_visualize = import ../nixupdate/nix-visualize.nix { pkgs=pkgs; };
+  makeWrapperArgs = [
+    "--prefix PATH : ${pkgs.lib.makeBinPath [ sbomnix repology_cli nix_visualize ]}"
+  ];
+
+  propagatedBuildInputs = [ 
+    sbomnix
+  ];
+
+  postInstall = ''
+    install -vD scripts/nixupdate/nix_outdated.py $out/bin/nix_outdated.py
+  '';
+
+  pythonImportsCheck = [ "sbomnix" ];
+}
diff --git a/scripts/nixupdate/nix_outdated.py b/scripts/nixupdate/nix_outdated.py
@@ -31,7 +31,6 @@
 ###############################################################################
 
 _LOG = logging.getLogger(LOGGER_NAME)
-_MYDIR = pathlib.Path(os.path.dirname(os.path.realpath(__file__)))
 
 ###############################################################################
 
@@ -95,7 +94,7 @@ def _run_repology_cli(sbompath):
     suffix = ".csv"
     with NamedTemporaryFile(delete=False, prefix=prefix, suffix=suffix) as f:
         cmd = (
-            "scripts/repology/repology_cli.py "
+            "repology_cli.py "
             f"--sbom_cdx={sbompath} --repository=nix_unstable --out={f.name}"
         )
         exec_cmd(cmd.split())

diff --git a/scripts/repology/README.md b/scripts/repology/README.md
@@ -23,6 +23,21 @@ Table of Contents
 
 ## Getting Started
 
+### Running as Nix Flake
+`repology_cli` can be run as a [Nix flake](https://nixos.wiki/wiki/Flakes) from the `tiiuae/sbomnix` repository:
+```bash
+# '--' signifies the end of argument list for `nix`.
+# '--help' is the first argument to `repology_cli`
+$ nix run github:tiiuae/sbomnix#repology_cli -- --help
+```
+
+or from a local repository:
+```bash
+$ git clone https://github.com/tiiuae/sbomnix
+$ cd sbomnix
+$ nix run .#repology_cli -- --help
+```
+
 ### Running from Nix Development Shell
 
 If you have nix flakes enabled, run:

diff --git a/scripts/repology/pyrate-limiter.nix b/scripts/repology/pyrate-limiter.nix
@@ -0,0 +1,28 @@
+# SPDX-FileCopyrightText: 2023 Technology Innovation Institute (TII)
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# TODO: this should be in nixpkgs
+
+{ nixpkgs ? <nixpkgs>
+, pkgs ? import nixpkgs {}
+, pythonPackages ? pkgs.python3Packages
+, lib ? pkgs.lib
+}:
+
+pythonPackages.buildPythonPackage rec {
+  version = "2.10.0";
+  pname = "pyrate-limiter";
+  format = "pyproject";
+
+  src = pkgs.fetchFromGitHub {
+    owner = "vutran1710";
+    repo = "PyrateLimiter";
+    rev = "v${version}";
+    hash = "sha256-CPusPeyTS+QyWiMHsU0ii9ZxPuizsqv0wQy3uicrDw0=";
+  };
+
+  propagatedBuildInputs = with pythonPackages; [
+    poetry-core
+  ];
+}
diff --git a/scripts/repology/repology_cli.nix b/scripts/repology/repology_cli.nix
@@ -0,0 +1,36 @@
+# SPDX-FileCopyrightText: 2023 Technology Innovation Institute (TII)
+#
+# SPDX-License-Identifier: Apache-2.0
+{
+  pkgs ? import <nixpkgs> {},
+  pythonPackages ? pkgs.python3Packages,
+}:
+
+pythonPackages.buildPythonPackage rec {
+  pname = "repology_cli";
+  version = pkgs.lib.removeSuffix "\n" (builtins.readFile ../../VERSION);
+  format = "setuptools";
+
+  src = ../../.;
+  sbomnix = import ../../default.nix { pkgs=pkgs; };
+  makeWrapperArgs = [
+    "--prefix PATH : ${pkgs.lib.makeBinPath [ sbomnix ]}"
+  ];
+
+  requests-ratelimiter = import ./requests-ratelimiter.nix { pkgs=pkgs; };
+
+  propagatedBuildInputs = [ 
+    sbomnix
+    requests-ratelimiter
+    pythonPackages.beautifulsoup4
+    pythonPackages.requests
+    pythonPackages.requests-cache
+    pythonPackages.packaging
+  ];
+
+  postInstall = ''
+    install -vD scripts/repology/repology_cli.py $out/bin/repology_cli.py
+  '';
+
+  pythonImportsCheck = [ "sbomnix" ];
+}