Merge branch 'master' into SurfaceUnits_csv

CadRevealComposer/Operations/HierarchyComposerConverter.cs

@@ -43,11 +43,7 @@ public static IReadOnlyList<HierarchyNode> ConvertToHierarchyNodes(CadRevealNode
         var maybeRefNoString = rvmNode.Attributes.GetValueOrNull("RefNo");
 
         RefNo? maybeRefNo = null;
-        if (!string.IsNullOrWhiteSpace(maybeRefNoString)
-            &&
-            // TEMP: 2022-08-25 Ignore ILTUBOF RefNos instead of actually handling them, as the database does not yet support representing this.
-            !maybeRefNoString.Trim().StartsWith("ILTUBOF", StringComparison.OrdinalIgnoreCase)
-           )
+        if (!string.IsNullOrWhiteSpace(maybeRefNoString))
         {
             maybeRefNo = RefNo.Parse(maybeRefNoString);
         }
@@ -74,6 +70,7 @@ public static IReadOnlyList<HierarchyNode> ConvertToHierarchyNodes(CadRevealNode
         {
             NodeId = ConvertUlongToUintOrThrowIfTooLarge(revealNode.TreeIndex),
             EndId = ConvertUlongToUintOrThrowIfTooLarge(GetLastIndexInChildrenIncludingSelf(revealNode)),
+            RefNoPrefix = maybeRefNo?.Prefix,
             RefNoDb = maybeRefNo?.DbNo,
             RefNoSequence = maybeRefNo?.SequenceNo,
             Name = rvmNode.Name,

HierarchyComposer.Tests/RefNoTests.cs

@@ -13,6 +13,7 @@ public class RefNoTests
     [TestCase("=123/")]
     [TestCase("=-123/321")] // Must not be negative
     [TestCase("=123/123/123")] // Two Numbers
+    [TestCase("PREFIX WITH SPACES=123/321")] // Prefix with spaces is not yet found in real data. Change this if needed.
     public void Parse_WhenGivenInvalidValues_ThrowsArgumentException(string invalidInput)
     {
         Assert.Throws<ArgumentException>(() => _ = RefNo.Parse(invalidInput));
@@ -25,14 +26,39 @@ public void Parse_WhenGivenNullValues_ThrowsArgumentNullException()
     }
 
     [Test]
-    [TestCase("=123/1337", 123, 1337)]
-    [TestCase("=1/0", 1, 0)]
-    [TestCase("=8129/0", 8129, 0)]
+    [TestCase("=123/1337", 123, 1337, "")]
+    [TestCase("=1/0", 1, 0, "")]
+    [TestCase("=8129/0", 8129, 0, "")]
+    [TestCase("ILTUBOF=8129/0", 8129, 0, "ILTUBOF")]
     public void Parse_WhenGivenValidValues_ReturnsRefNoWithExpectedDbAndSeq(string validInput, int expectedDb,
-        int expectedSequence)
+        int expectedSequence, string expectedPrefix)
     {
         var result = RefNo.Parse(validInput);
         Assert.That(result.DbNo, Is.EqualTo(expectedDb));
         Assert.That(result.SequenceNo, Is.EqualTo(expectedSequence));
+        Assert.That(result.Prefix, Is.EqualTo(expectedPrefix));
+    }
+
+    [Test]
+    [TestCase("ILTUBOF", "ILTUBOF")]
+    [TestCase("123", "123")]
+    [TestCase("", "")]
+    [TestCase(null, "")]
+    public void Constructor_WhenGivenValidValues_ReturnsExpectedRefNo(string actualPrefix, string expectedPrefix)
+    {
+        var refNo = new RefNo(actualPrefix, 123,321);
+        Assert.That(refNo.Prefix, Is.EqualTo(expectedPrefix));
+    }
+
+    [Test]
+    [TestCase("PREFIX WITH SPACES")]
+    [TestCase(" ")]
+    [TestCase(" ILTUBOF")] // Expect trimmed input
+    public void Constructor_WhenGivenInvalidValues_Throws(string invalidPrefix)
+    {
+        Assert.Throws<ArgumentException>(() =>
+        {
+            _ = new RefNo(invalidPrefix, 123, 321);
+        });
     }
 }

HierarchyComposer/Functions/DatabaseComposer.cs

@@ -72,6 +72,7 @@ public void ComposeDatabase(IReadOnlyList<HierarchyNode> inputNodes, string outp
         {
             Id = inputNode.NodeId,
             EndId = inputNode.EndId,
+            RefNoPrefix = inputNode.RefNoPrefix,
             RefNoDb = inputNode.RefNoDb,
             RefNoSequence = inputNode.RefNoSequence,
             Name = inputNode.Name,
@@ -149,7 +150,8 @@ public void ComposeDatabase(IReadOnlyList<HierarchyNode> inputNodes, string outp
             cmd.CommandText = "CREATE INDEX PDMSEntries_Key_index ON PDMSEntries (Key)";
             cmd.ExecuteNonQuery();
             cmd.CommandText = "CREATE INDEX Nodes_Name_index ON Nodes (Name)";
-            cmd.CommandText = "CREATE INDEX Nodes_RefNo_Index ON Nodes (RefNoDb, RefNoSequence)";
+            cmd.ExecuteNonQuery();
+            cmd.CommandText = "CREATE INDEX Nodes_RefNo_Index ON Nodes (RefNoPrefix, RefNoDb, RefNoSequence)";
             cmd.ExecuteNonQuery();
 
             transaction.Commit();
@@ -158,18 +160,22 @@ public void ComposeDatabase(IReadOnlyList<HierarchyNode> inputNodes, string outp
 
         MopTimer.RunAndMeasure("VACUUM Database", _logger, () =>
         {
+#if DEBUG
+            // Ignore in debug mode to run faster
+            return;
+#else
             // Vacuum completely recreates the database but removes all "Extra Data" from it.
             // Its a quite slow operation but might fix the "First query is super slow issue" on the hierarchy service.
             using var vacuumCmds = new SQLiteCommand(connection);
 
             vacuumCmds.CommandText = "PRAGMA page_count";
-            var pageCountBeforeVacuum = (Int64) vacuumCmds.ExecuteScalar();
+            var pageCountBeforeVacuum = (Int64)vacuumCmds.ExecuteScalar();
             var timer = Stopwatch.StartNew();
             // Vacuum the database. This is quite slow!
             vacuumCmds.CommandText = "VACUUM";
             vacuumCmds.ExecuteNonQuery();
             vacuumCmds.CommandText = "PRAGMA page_count";
-            var pageCountAfterVacuum = (Int64) vacuumCmds.ExecuteScalar();
+            var pageCountAfterVacuum = (Int64)vacuumCmds.ExecuteScalar();
 
             // Disable auto_vacuum explicitly as we expect no more data to be written to the database after this.
             vacuumCmds.CommandText = "PRAGMA auto_vacuum = NONE";
@@ -184,6 +190,7 @@ public void ComposeDatabase(IReadOnlyList<HierarchyNode> inputNodes, string outp
             // FUTURE: Consider if we should disable VACUUM in dev builds if its too slow, its not really needed there.
             Console.WriteLine(
                 $"VACUUM finished in {timer.Elapsed}. Reduced size from {pageCountBeforeVacuum} to {pageCountAfterVacuum}");
+#endif
         });
 
         // ReSharper restore AccessToDisposedClosure

HierarchyComposer/Model/HierarchyNode.cs

@@ -6,6 +6,8 @@ public class HierarchyNode
 {
     public uint NodeId { get; set; }
     public uint EndId { get; set; }
+
+    public string? RefNoPrefix { get; set; }
     public int? RefNoDb { get; set; }
     public int? RefNoSequence { get; set; }
     public string Name { get; set; } = "";

HierarchyComposer/Model/Node.cs

@@ -8,9 +8,8 @@ public class Node
 {
     [DatabaseGenerated(DatabaseGeneratedOption.None)]
     public uint Id { get; init; }
-
     public uint EndId { get; init; }
-
+    public string? RefNoPrefix { get; init; }
     public int? RefNoDb { get; init; }
     public int? RefNoSequence { get; init; }
 
@@ -31,10 +30,11 @@ public class Node
 
     public void RawInsert(SQLiteCommand command)
     {
-        command.CommandText = "INSERT INTO Nodes (Id, EndId, RefNoDb, RefNoSequence, Name, HasMesh, ParentId, TopNodeId, AABBId, DiagnosticInfo) VALUES (@Id, @EndId, @RefNoDb, @RefNoSequence, @Name, @HasMesh, @ParentId, @TopNodeId, @AABBId, @DiagnosticInfo);";
+        command.CommandText = "INSERT INTO Nodes (Id, EndId, RefNoPrefix, RefNoDb, RefNoSequence, Name, HasMesh, ParentId, TopNodeId, AABBId, DiagnosticInfo) VALUES (@Id, @EndId, @RefNoPrefix, @RefNoDb, @RefNoSequence, @Name, @HasMesh, @ParentId, @TopNodeId, @AABBId, @DiagnosticInfo);";
         command.Parameters.AddRange(new[] {
             new SQLiteParameter("@Id", Id),
             new SQLiteParameter("@EndId", EndId),
+            new SQLiteParameter("@RefNoPrefix", RefNoPrefix),
             new SQLiteParameter("@RefNoDb", RefNoDb),
             new SQLiteParameter("@RefNoSequence", RefNoSequence),
             new SQLiteParameter("@Name", Name),

HierarchyComposer/Model/RefNo.cs

@@ -7,36 +7,56 @@
 
 /// <summary>
 /// In the PDMS format every element has an ID, which is unique to an object.
-/// It consists of two parts, where the firs is a DatabaseNumber
-/// and the second a SequenceNumber
-/// It will look like this in the data: "=123/456"
+/// Some parts do not have their own Ids but are attached to a part, they are identified as ILTUBOF See remarks..
+/// It consists of three parts.
+/// 1. OPTIONAL: Prefix of a single word. Such as ILTUBOF
+/// 2. Required: A DatabaseNumber
+/// 3. Required: A SequenceNumber
+/// It will look like this in the data: "=123/456" or "ILTUBOF=123/456"
 /// </summary>
-[DebuggerDisplay("={DbNo}/{SequenceNo}")]
-public class RefNo
+/// <remarks>
+/// Prefixes such as ILTUBOF mean Implicit Leaving TUBi OF =RefNoDb/RefNoSequence, and indicates that this is
+/// and implicitly connecting tube that connects from the previous part (such as a bend or flange), to the next part (bend or flange etc.).
+/// </remarks>
+[DebuggerDisplay("{Prefix}={DbNo}/{SequenceNo}")]
+public partial class RefNo
 {
+    /// <summary>
+    /// Prefix: In the current known data this is either empty or ILTUBOF
+    /// Its really a "Query" for the data, in PDMS. But we do not support queries (yet) so we store it as a "dumb" prefix.
+    /// </summary>
+    /// <remarks>
+    /// This should never be null as it would make DB queries a lot more complicated ((NULL = NULL) is UNKNOWN in SQL)
+    /// </remarks>
+    public string Prefix { get; set; }
     public int DbNo { get; }
     public int SequenceNo { get; }
 
     public override string ToString()
     {
-        return $"={DbNo}/{SequenceNo}";
+        return $"{Prefix}={DbNo}/{SequenceNo}";
     }
 
-    public RefNo(int dbNo, int sequenceNo)
+    public RefNo(string? prefix, int dbNo, int sequenceNo)
     {
+        if (!string.IsNullOrEmpty(prefix) && !PrefixValidRegex.IsMatch(prefix))
+        {
+            throw new ArgumentException($"Prefix \"{prefix}\" is unexpected, is this a valid prefix? If so update the code and tests.");
+        }
+
+        Prefix = prefix ?? string.Empty;
         DbNo = dbNo;
         SequenceNo = sequenceNo;
     }
 
-    /// <summary>
-    /// Matches strings with this pattern: =123/456, with one capturing group () for each number.
-    /// </summary>
-    private static readonly Regex RefNoRegex = new Regex(@"^=(\d+)\/(\d+)$", RegexOptions.Compiled);
+    private static readonly Regex RefNoRegex = RefNoMatchingRegex();
+    // Ensure the regex is only
+    private static readonly Regex PrefixValidRegex = new Regex("^\\w+$", RegexOptions.Compiled);
 
     /// <summary>
-    /// Parse a RefNo string into a RefNo
+    /// Parse a RefNo-string into a RefNo
     /// </summary>
-    /// <param name="refNo">Expects a "=123/456" formatted string</param>
+    /// <param name="refNo">Expects a "=123/456" or "PREFIX=123/321" formatted string</param>
     /// <returns></returns>
     /// <exception cref="ArgumentException">If the input format is not correct</exception>
     /// <exception cref="ArgumentNullException"></exception>
@@ -46,18 +66,29 @@ public static RefNo Parse(string refNo)
         if (refNo == null)
             throw new ArgumentNullException(nameof(refNo));
 
-        var match = RefNoRegex.Match(refNo);
+        var match = RefNoRegex.Match(refNo.Trim());
 
         if (!match.Success)
-            throw new ArgumentException($"Expected format '=123/321' '(=uint/uint)', was '{refNo}'", nameof(refNo));
+            throw new ArgumentException($"Expected format 'prefix=123/321' '(string?=uint/uint)' (prefix is optional), was '{refNo}'", nameof(refNo));
 
         // Regex Group 0 is the entire match.
-        var dbNo = int.Parse(match.Groups[1].Value);
-        var sequenceNo = int.Parse(match.Groups[2].Value);
+        var prefixParsed = match.Groups[1].Value;
+        var dbNo = int.Parse(match.Groups[2].Value);
+        var sequenceNo = int.Parse(match.Groups[3].Value);
 
         if (dbNo < 0 || sequenceNo < 0)
             throw new ArgumentException($"Expected positive values, was '{refNo}'", nameof(refNo));
 
-        return new RefNo(dbNo, sequenceNo);
+        // Save empty prefixes as string.empty
+        var prefix = string.IsNullOrWhiteSpace(prefixParsed) ? String.Empty : prefixParsed;
+
+        return new RefNo(prefix, dbNo, sequenceNo);
     }
+
+    /// <summary>
+    /// Matches strings with this pattern: PREFIX?=123/456, with one capturing group () for each part.
+    /// Prefix is optional
+    /// </summary>
+    [GeneratedRegex("^(\\w+)?=(\\d+)\\/(\\d+)$", RegexOptions.Compiled)]
+    private static partial Regex RefNoMatchingRegex();
 }