[Core] Add tool to extract all code snippets from a document

build.gradle

@@ -7,7 +7,7 @@ plugins {
 }
 
 group 'net.signbit.samx'
-version '0.4.4'
+version '0.4.5'
 
 sourceCompatibility = 1.8
 
@@ -55,6 +55,7 @@ def scripts = ['tokenize'       : 'net.signbit.samx.Tokenize',
                'to_xml'         : 'net.signbit.samx.ConvertToXml',
                'to_html'        : 'net.signbit.samx.ConvertToHtml',
                'pretty_print'   : 'net.signbit.samx.PrettyPrint',
+               'extract_code'   : 'net.signbit.samx.ExtractCode',
                'generate_header': 'net.signbit.samx.literate.GenerateHeader',
 ]
 

doc/getting_started.md

@@ -79,6 +79,8 @@ There are several applications bundled in the distribution package:
 
 ```shell script
 $ ls bin | cat
+extract_code
+extract_code.bat
 generate_header
 generate_header.bat
 pretty_print
@@ -95,7 +97,7 @@ to_xml.bat
 
 The applications can run on either UNIX or Windows.
 
-Pretty-print
+Pretty print
 ------------
 
 `pretty_print` parses the input document and writes out a copy in canonical form: it normalizes white space by
@@ -110,6 +112,12 @@ To-XML
 
 `to_xml` converts the input document into the corresponding XML format. It should support most of the SAMx constructs.
 
+Extract Code
+------------
+
+`extract_code` parses the input document and extracts all the code blocks into individual files. This is useful if the
+document contains embedded code or diagrams which need to be rendered with external tools.
+
 Debug Utilities
 ---------------
 

doc/samx_language.samx

@@ -76,9 +76,11 @@ chapter: SAMx Tools
 
    * pretty_print
 
+   * extract_code
+
    * to_xml
 
-   section: pretty_print
+   section: Pretty Print
 
       Pretty print is a tool that parses the SAMx document and then
       recreates it by visiting every node. The pretty printer does not
@@ -94,6 +96,13 @@ chapter: SAMx Tools
       diagnostic. Otherwise the tool will provide an indication of the
       first mismatch location.
 
+   section: Extract Code
+
+      Extract code parses the input document and extracts all the code
+      blocks into individual files. This is useful if the document
+      contains embedded code or diagrams which need to be rendered with
+      external tools.
+
 chapter: Building SAMx from source
 
    The two prerequites are Git, for downloading the source code and Java
@@ -105,21 +114,17 @@ chapter: Building SAMx from source
    {release}(:https://github.com/0x8000-0000/samx-java/releases) from
    GitHub.
 
-   Simply run \"./gradlew clean build assembleDist\" to build, test and
+   Simply run \"./gradlew clean build assemble\" to build, test and
    create a binary distribution. The binary distribution is generated in
    \"build/distributions/\" directory.
 
    section: Building this Document from Source
 
-      This document is available in SAMx source for in the doc
-      subdirectory of a source or a binary distribution.
-
-      Run \"./bin/to_xml -i doc/samx_language.samx -b -o samx_book.xml\"
-      to convert it.
-
-      Then run \"jing -f docbook.rng samx_book.xml\" to validate it.
+      This document is available in SAMx source tree and also in the
+      \"docs\" subdirectory in a binary distribution.
 
-      Then run \"dblatex samx_book.xml\" to convert this book to PDF.
+      Run \"dblatex docs/samx_language.dbk\" to convert this book to
+      PDF.
 
 chapter: Frequently Asked Questions
 

src/main/java/net/signbit/samx/ExtractCode.java

@@ -0,0 +1,83 @@
+/*
+   Copyright 2020 Florin Iucha
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package net.signbit.samx;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.commons.cli.*;
+
+import net.signbit.samx.visitors.EmbeddedCodeVisitor;
+
+public class ExtractCode
+{
+   public static void main(String[] args) throws IOException
+   {
+      Options options = new Options();
+
+      Option input = new Option("i", "input", true, "input file path");
+      input.setRequired(true);
+      options.addOption(input);
+
+      Option output = new Option("o", "output", true, "output file path");
+      output.setRequired(true);
+      options.addOption(output);
+
+      CommandLineParser cmdLine = new DefaultParser();
+      HelpFormatter helpFmt = new HelpFormatter();
+
+      try
+      {
+         CommandLine cmd = cmdLine.parse(options, args);
+
+         Parser.Result result = Parser.parse(cmd.getOptionValue("input"));
+         if (result.errorCount > 0)
+         {
+            System.err.print("Failed to parse input file " + cmd.getOptionValue("input"));
+            System.exit(10);
+         }
+
+         File outputDir = null;
+         if (cmd.getOptionValue("output") != null)
+         {
+            outputDir = new File(cmd.getOptionValue("output"));
+            if (! outputDir.isDirectory())
+            {
+               System.err.println("Specified output path " + cmd.getOptionValue("output") + " is not a directory.");
+               System.exit(5);
+            }
+         }
+         else
+         {
+            outputDir = new File(".");
+         }
+
+         EmbeddedCodeVisitor visitor = new EmbeddedCodeVisitor(result.tokens, outputDir);
+
+         visitor.visit(result.document);
+
+         System.exit(0);
+      }
+      catch (ParseException pe)
+      {
+         System.err.println(pe.getMessage());
+         helpFmt.printHelp("Renderer", options);
+      }
+
+      System.exit(1);
+   }
+}

src/main/java/net/signbit/samx/visitors/AttributeVisitor.java

@@ -47,6 +47,7 @@ public String getReference()
    private String referenceAttribute = null;
    private String citationAttribute = null;
 
+
    @Override
    public Void visitClassAttr(SamXParser.ClassAttrContext ctx)
    {
@@ -214,4 +215,21 @@ public String toPlainString()
       return builder.toString();
    }
 
+   @Override
+   public Void visitMetadata(SamXParser.MetadataContext ctx)
+   {
+      for (SamXParser.AttributeContext ac : ctx.attribute())
+      {
+         visit(ac);
+      }
+      return null;
+   }
+
+   @Override
+   public Void visitBlockMetadata(SamXParser.BlockMetadataContext ctx)
+   {
+      visitMetadata(ctx.metadata());
+      return null;
+   }
+
 }

src/main/java/net/signbit/samx/visitors/EmbeddedCodeVisitor.java

@@ -0,0 +1,113 @@
+/*
+   Copyright 2020 Florin Iucha
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package net.signbit.samx.visitors;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+
+import org.antlr.v4.runtime.BufferedTokenStream;
+
+import net.signbit.samx.parser.SamXParser;
+import net.signbit.samx.parser.SamXParserBaseVisitor;
+
+public class EmbeddedCodeVisitor extends SamXParserBaseVisitor<StringBuilder>
+{
+   private final BufferedTokenStream tokenStream;
+   private final File parentDir;
+
+   public EmbeddedCodeVisitor(BufferedTokenStream tokenStream, File parentDir)
+   {
+      this.tokenStream = tokenStream;
+      this.parentDir = parentDir;
+   }
+
+   @Override
+   public StringBuilder visitDocument(SamXParser.DocumentContext ctx)
+   {
+      for (SamXParser.BlockContext bc : ctx.block())
+      {
+         visit(bc);
+      }
+
+      return null;
+   }
+
+   @Override
+   public StringBuilder visitTypedBlock(SamXParser.TypedBlockContext ctx)
+   {
+      for (SamXParser.BlockContext bc : ctx.block())
+      {
+         visit(bc);
+      }
+
+      return null;
+   }
+
+   @Override
+   public StringBuilder visitCodeBlock(SamXParser.CodeBlockContext ctx)
+   {
+      AttributeVisitor attributeVisitor = new AttributeVisitor();
+      attributeVisitor.visit(ctx.metadata());
+
+      final String fileStem = attributeVisitor.getId();
+      final String fileExtension = ctx.language.getText();
+
+      final File outputFile = new File(parentDir, fileStem + "." + fileExtension);
+
+      try
+      {
+         System.out.println("Writing " + outputFile.getCanonicalPath());
+
+         final FileWriter writer = new FileWriter(outputFile);
+
+         /*
+          * find the leftmost code line - that will become column 0
+          */
+         int codeBlockIndent = 65536;
+         for (SamXParser.ExternalCodeContext ecc : ctx.externalCode())
+         {
+            final int codeLineIndent = VisitorUtils.getTokenIndent(ecc, tokenStream);
+            if (codeBlockIndent > codeLineIndent)
+            {
+               codeBlockIndent = codeLineIndent;
+            }
+         }
+
+         for (SamXParser.ExternalCodeContext ecc : ctx.externalCode())
+         {
+            final int codeLineIndent = VisitorUtils.getTokenIndent(ecc, tokenStream);
+
+            for (int ii = codeBlockIndent; ii < codeLineIndent; ++ ii)
+            {
+               writer.append(' ');
+            }
+
+            writer.append(ecc.EXTCODE().getText());
+            writer.append('\n');
+         }
+
+         writer.close();
+      }
+      catch (IOException ioe)
+      {
+         System.err.println("Cannot open file " + outputFile.getAbsolutePath() + " for writing: " + ioe.getMessage());
+      }
+
+      return null;
+   }
+}

src/main/java/net/signbit/samx/visitors/PrettyPrinterVisitor.java

@@ -674,6 +674,7 @@ public StringBuilder visitCodeBlock(SamXParser.CodeBlockContext ctx)
       builder.append("```(");
       builder.append(visit(ctx.language));
       builder.append(")");
+      builder.append(visitMetadata(ctx.metadata()));
       builder.append('\n');
 
       indentLevel++;

src/test/resources/sam/embedded_code.sam

@@ -6,7 +6,7 @@ markup:
 
 result:
 
-   ```(xml)
+   ```(xml)(#example)
        <section xml:lang="es">
           <citation>Mother McCree</citation>
           <title>Greeting</title>