Pull Request from Sam Gresh @ NACHC- Match Executable

.gitignore

@@ -1,3 +1,7 @@
+tests/exe_test
+commit.bat
+.idea
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

DCCExecutable.py

@@ -0,0 +1,206 @@
+import validate, match, linkids, dataownerids
+import json
+import wx
+from multiprocessing import freeze_support
+from importlib import resources
+import os
+import socket
+import subprocess
+import atexit
+import time
+import sys
+
+#pyinstaller DCCExecutable.py  --add-data anonlink-entity-service;anonlink-entity-service --add-data config.json;.
+
+
+if getattr(sys, 'frozen', False) and hasattr(sys, '_MEIPASS'):
+    os.chdir(sys._MEIPASS)
+print(os.getcwd())
+atexit.register(subprocess.run, "docker-compose -p anonlink -f anonlink-entity-service/tools/docker-compose.yml down")
+atexit.register(os.remove, "results.json")
+try:
+    os.remove("results.json")
+except:
+    pass
+
+with open("config.json") as f:
+    config = json.load(f)
+    ownerNames = config['systems']
+
+class CSVManager(wx.Frame):
+    def __init__(self, *args, **kwargs):
+        super(CSVManager, self).__init__(*args, **kwargs)
+        self.schemaDir = ""
+        self.inboxDir = ""
+        self.outputDir = ""
+        self.enter_owner_names_text = None
+        self.owner_names_input = None
+        self.open_schema_text = None
+        self.open_inbox_text = None
+        self.open_output_text = None
+        self.threshold_text = None
+        self.threshold_input = None
+        self.inbox_text = None
+        self.port_text = None
+        self.match_text = None
+        self.InitUI()
+        self.PingService(True)
+
+    def InitUI(self):
+        panel = wx.Panel(self)
+
+        hbox = wx.BoxSizer()
+        sizer = wx.GridSizer(8, 2, 10, 300)
+
+        schema_btn = wx.Button(panel, label='Select Schema Folder')
+        inbox_btn = wx.Button(panel, label='Select Inbox Folder')
+        output_btn = wx.Button(panel, label='Select Output Folder')
+        validate_btn = wx.Button(panel, label='Validate Inbox')
+        start_service_btn = wx.Button(panel, label='Start Service')
+        match_btn = wx.Button(panel, label='Match Records')
+
+        self.enter_owner_names_text = wx.StaticText(panel, label="Enter Data Owner Names (comma seperated)")
+        self.owner_names_input = wx.TextCtrl(panel, value=str(ownerNames).replace("[", "").replace("]", "").replace("'", ""))
+        self.open_schema_text = wx.StaticText(panel, label=config['schema_folder'])
+        self.open_inbox_text = wx.StaticText(panel, label=config['inbox_folder'])
+        self.open_output_text = wx.StaticText(panel, label=config['output_folder'])
+        self.threshold_text = wx.StaticText(panel, label="Enter matching threshold")
+        self.threshold_input = wx.TextCtrl(panel, value=str(config["matching_threshold"]))
+        self.inbox_text = wx.StaticText(panel, label="")
+        self.port_text = wx.StaticText(panel, label="")
+        self.match_text = wx.StaticText(panel, label="")
+
+        sizer.AddMany([self.enter_owner_names_text, self.owner_names_input, self.open_schema_text, schema_btn, self.open_inbox_text, inbox_btn, self.open_output_text, output_btn, self.threshold_text, self.threshold_input, self.inbox_text, validate_btn, self.port_text, start_service_btn, self.match_text, match_btn])
+
+        hbox.Add(sizer, 0, wx.ALL, 15)
+        panel.SetSizer(hbox)
+
+        schema_btn.Bind(wx.EVT_BUTTON, self.OnOpenSchema)
+        inbox_btn.Bind(wx.EVT_BUTTON, self.OnOpenInbox)
+        output_btn.Bind(wx.EVT_BUTTON, self.OnOpenOutput)
+        validate_btn.Bind(wx.EVT_BUTTON, self.StartValidate)
+        start_service_btn.Bind(wx.EVT_BUTTON, self.StartService)
+        match_btn.Bind(wx.EVT_BUTTON, self.StartMatch)
+
+        self.owner_names_input.Bind(wx.EVT_TEXT, self.UpdateOwners)
+        self.threshold_input.Bind(wx.EVT_TEXT, self.UpdateThreshold)
+
+        self.SetSize((850, 300))
+        self.SetTitle('Messages')
+        self.Centre()
+
+    def PingService(self, init):
+        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        result = sock.connect_ex(('127.0.0.1', 8851))
+        if result == 0:
+            self.port_text.SetLabel("Port is open")
+            self.port_text.SetForegroundColour((0, 150, 0))
+            sock.close()
+            return True
+        else:
+            self.port_text.SetLabel("Port is not open")
+            if init:
+                self.port_text.SetForegroundColour((150, 0, 0))
+            else:
+                self.port_text.SetForegroundColour((150, 150, 0))
+            sock.close()
+            return False
+
+
+    def StartService(self, event):
+        subprocess.Popen("docker-compose -p anonlink -f anonlink-entity-service/tools/docker-compose.yml up")
+        while not self.PingService(False):
+            pass
+
+    def StartValidate(self, event):
+        msg = validate.validate()
+        self.inbox_text.SetLabel(msg)
+        if msg == "All necessary input is present":
+            self.inbox_text.SetForegroundColour((0, 150, 0))
+        else:
+            self.inbox_text.SetForegroundColour((150, 0, 0))
+
+    def StartMatch(self, event):
+        self.match_text.SetLabel("Matching Entries...")
+        self.match_text.SetForegroundColour((150, 150, 0))
+        self.Update()
+        time.sleep(5)
+        match.match()
+        linkids.linkids()
+        dataownerids.data_owner_ids()
+        self.match_text.SetLabel(f"Linkids written to {config['output_folder']}")
+        self.match_text.SetForegroundColour((0, 150, 0))
+        self.Update()
+        try:
+            os.remove("results.json")
+        except:
+            pass
+
+    def UpdateJson(self):
+        global config
+        with open("config.json", "w") as f:
+            json.dump(config, f)
+        with open("config.json") as f:
+            config = json.load(f)
+            ownerNames = [owner + ", " for owner in config['systems']]
+
+    def UpdateOwners(self, event):
+        config['systems'] = [owner.strip(" ") for owner in self.owner_names_input.GetLineText(0).split(",")]
+        self.UpdateJson()
+
+
+    def UpdateThreshold(self, event):
+        config['matching_threshold'] = float(self.threshold_input.GetLineText(0))
+        self.UpdateJson()
+
+
+    def OnOpenSchema(self, event):
+        with wx.DirDialog(self, "Choose Schema Directory", style=wx.FD_OPEN | wx.DD_DIR_MUST_EXIST) as dirDialog:
+            if dirDialog.ShowModal() == wx.ID_CANCEL:
+                return  # the user changed their mind
+
+            # Proceed loading the file chosen by the user
+            self.schemaDir = dirDialog.GetPath()
+            self.open_schema_text.SetLabel(self.schemaDir)
+            config['schema_folder'] = self.schemaDir
+            self.UpdateJson()
+
+
+    def OnOpenInbox(self, event):
+        with wx.DirDialog(self, "Choose Inbox Directory", style=wx.FD_OPEN | wx.DD_DIR_MUST_EXIST) as dirDialog:
+            if dirDialog.ShowModal() == wx.ID_CANCEL:
+                return  # the user changed their mind
+
+            # Proceed loading the file chosen by the user
+            self.inboxDir = dirDialog.GetPath()
+            self.open_inbox_text.SetLabel(self.inboxDir)
+            config['inbox_folder'] = self.inboxDir
+            self.UpdateJson()
+
+    def OnOpenOutput(self, event):
+        with wx.DirDialog(self, "Choose Output Directory", style=wx.FD_OPEN | wx.DD_DIR_MUST_EXIST) as dirDialog:
+            if dirDialog.ShowModal() == wx.ID_CANCEL:
+                return  # the user changed their mind
+
+            # Proceed loading the file chosen by the user
+            self.outputDir = dirDialog.GetPath()
+            self.open_output_text.SetLabel(self.outputDir)
+            config['output_folder'] = self.outputDir
+            config['matching_results_folder'] = self.outputDir
+            self.UpdateJson()
+
+
+
+
+def main():
+    app = wx.App()
+    ex = CSVManager(None)
+    ex.Show()
+    app.MainLoop()
+
+
+
+if __name__ == '__main__':
+    freeze_support()
+    main()
+

README.md

@@ -89,8 +89,43 @@ Linkage Agent Tools contains a unit test suite. Tests can be run with the follow
 
 `python -m pytest`
 
+# Linkage Agent Tools Executables
+A Widows executable version of the tools with a GUI is availible. Docker Desktop is the only dependency of the GUI tool. Command line and GUI tools are interoperable. Executables built using pyinstaller version 4.2.
+
+## Basic User Instructions
+Download the DCCexecutable.zip file from https://github.com/Sam-Gresh/linkage-agent-tools/releases/tag/v0.0.3. Unzip the file to a new directory. In the folder DCCExecutable, run the executable DCCexecutable and select the correct schema, inbox, and output directories. If an error occurs, the directories can be set manually from config.json. Make sure docker is running, then click start service. Place the garbled zip files in the specified inbox directory, and set the "Data providers" field. Validate the inbox, then click Match. All csv outputs should appear in the specified output directory after some time.
+
+## Change Log
+### match.py
+- Moved majority of code into callable function
+
+### validate.py
+- Moved majority of code into callable function
+- Moved argparse code into \_\_name\_\_ == "\_\_main\_\_" check
+- Function returns messages instead of printing
+
+### linkids.py
+- Moved majority of code into callable function
+- Moved argparse code into \_\_name\_\_ == "\_\_main\_\_" check
+- Function returns messages instead of printing
+
+## Additions
+### DCCExecutable.py
+- WxPython GUI wrapper for functions listed above
+- Able to be built into a one-folder executable using pyinstaller
+- Currently does not include Schema files inside the built folder
+- Runs multiprocessing.freeze_support() to enable multiprocessing in the built executable.
+
+
+## Build Instructions
+Clone the repository. From the cloned directory run the following commands:  
+`pip install -r requirements.txt`  
+`pyinstaller DCCExecutable.py  --add-data anonlink-entity-service;anonlink-entity-service --add-data config.json;.`
+
+The built executable will appear in /dist.
+
 ## Notice
 
 Copyright 2020 The MITRE Corporation.
 
-Approved for Public Release; Distribution Unlimited. Case Number 19-2008
+Approved for Public Release; Distribution Unlimited. Case Number 19-2008

anonlink-entity-service/.azurePipeline/k8s_get_results.yaml.tmpl

@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: $POD_NAME
+  labels:
+    deployment: $DEPLOYMENT_NAME
+spec:
+  restartPolicy: Never
+  volumes:
+    - name: results
+      persistentVolumeClaim:
+        claimName: $PVC
+  containers:
+    - name: resultpod
+      image: python
+      command: ["sleep", "3600"]
+      volumeMounts:
+        - mountPath: /mnt
+          name: results

anonlink-entity-service/.azurePipeline/k8s_test_job.yaml.tmpl

@@ -0,0 +1,37 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: $DEPLOYMENT_NAME-test
+  labels:
+    jobgroup: anonlink-integration-test
+    deployment: $DEPLOYMENT_NAME
+spec:
+  completions: 1
+  parallelism: 1
+  backoffLimit: 0
+  template:
+    metadata:
+      labels:
+        jobgroup: anonlink-integration-test
+        deployment: $DEPLOYMENT_NAME
+    spec:
+      securityContext:
+        runAsUser: 0
+        fsGroup: 0
+      restartPolicy: Never
+      volumes:
+        - name: results
+          persistentVolumeClaim:
+            claimName: $PVC
+      containers:
+        - name: entitytester
+          image: $TEST_E2E_IMAGE_NAME_WITH_TAG
+          imagePullPolicy: Always
+          env:
+            - name: SERVER
+              value: http://$SERVICE
+          command: ["dockerize", "-wait", "http://$SERVICE/api/v1/status", "-timeout", "5m", "python", "-m", "pytest", "-n", "4", "e2etests/tests", "-x", "--junit-xml=/mnt/results.xml"]
+          volumeMounts:
+            - mountPath: /mnt
+              name: results
+

anonlink-entity-service/.azurePipeline/k8s_test_pvc.yaml.tmpl

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: $PVC
+  labels:
+    jobgroup: anonlink-integration-test
+    deployment: $DEPLOYMENT_NAME
+spec:
+  storageClassName: gp2
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 1Gi