Merge pull request #42 from f5devcentral/1.2.0

1.2.0

CHANGELOG.md

@@ -13,11 +13,21 @@ Check [Keep a Changelog](http://keepachangelog.com/) for recommendations on how
 
 ### Changed
 
-- [RFE] include xml stats in extraction #15 (pending-researching)
+- [RFE] include xml stats in extraction #15 (researching)
 - main README.md/documentation updates
 
 ### Fixed
 
+
+---
+
+## [1.2.0] - (06.19.2023)
+
+### Fixed
+
+- [BUG] custom http monitor missing from abstration #40
+- [BUG] udf lab as3/acc/fast ucs bug on exploer #41
+
 ---
 
 ## [1.1.3] - (05.25.2023)

package.json

@@ -2,7 +2,7 @@
   "name": "f5-corkscrew",
   "description": "extracting tmos config",
   "author": "F5DevCentral",
-  "version": "1.1.3",
+  "version": "1.2.0",
   "license": "Apache-2.0",
   "homepage": "https://github.com/f5devcentral/f5-corkscrew#readme",
   "main": "dist/index.js",

src/deepParse.ts

@@ -547,10 +547,24 @@ export async function parseDeep(obj: any, rx: RegExTree) {
         const monitorRx = body.match(rx.ltm.pool.monitors)
 
         if (monitorRx) {
-            // monitors parse situation 1.  "requirement=>ALL", list of monitors
-            // monitor /Common/app1_tcp_half_open_quick_monitor and /Common/http_head_f5 and /Common/http2_head_f5 and /Common/http and /Common/tcp_half_open
+
             body = body.replace(monitorRx[0], '');
-            sObj.monitor = monitorRx[1].split(/ and /);
+
+            if (monitorRx[1].startsWith('{')) {
+
+                // monitor parse situation 1.  "min x of {"
+                // monitor min 2 of { /Common/gateway_icmp /Common/http /Common/http2 }
+                //  so we will look for "{" and parse accordingly
+                const monitors = monitorRx[1].replace(/({ | })/gm, '').split(' ');
+                sObj.monitor = monitors;
+
+            } else {
+
+                // monitors parse situation 2.  "requirement=>ALL", list of monitors
+                // monitor /Common/app1_tcp_half_open_quick_monitor and /Common/http_head_f5 and /Common/http2_head_f5 and /Common/http and /Common/tcp_half_open
+                sObj.monitor = monitorRx[1].split(/ and /);
+            }
+
         }
 
         const remaining = keyValuePairs(body)
@@ -588,7 +602,35 @@ export async function parseDeep(obj: any, rx: RegExTree) {
         sObj;
 
 
+    } else if (obj.ltm?.policy) {
+
+        const key = Object.keys(obj.ltm.policy)[0]; // only one policy at this point
+        let body = obj.ltm.policy[key];
+        obj.ltm.policy[key] = { line: body };   // re-assign the original config string
+        const sObj = obj.ltm.policy[key];
+
+
+    } else if (obj.ltm?.persistence) {
+
+        const type = Object.keys(obj.ltm.persistence)[0]; // only one policy at this point
+        const key = Object.keys(obj.ltm.persistence[type])[0];
+        let body = obj.ltm.persistence[type][key];
+        obj.ltm.persistence[type][key] = { line: body };   // re-assign the original config string
+        const sObj = obj.ltm.persistence[type][key];
+
+        const remaining = keyValuePairs(body)
+        deepmergeInto(sObj, remaining);
+        sObj;
+
+    } else if (obj.ltm?.profile) {
+
+        const type = Object.keys(obj.ltm.profile)[0]; // only one policy at this point
+        const key = Object.keys(obj.ltm.profile[type])[0];
+        let body = obj.ltm.profile[type][key];
+        obj.ltm.profile[type][key] = { line: body };   // re-assign the original config string
+        const sObj = obj.ltm.profile[type][key];
 
+        // todo: deep parse the rest of this object
 
     } else if (obj.ltm?.snatpool) {
 
@@ -624,6 +666,7 @@ export async function parseDeep(obj: any, rx: RegExTree) {
 
         const nameRx = key.match(rx.name);
         sObj.name = nameRx.groups.name;
+        sObj.type = monType;
         partitionFolder(sObj, nameRx.groups.partition);
 
         const remaining = keyValuePairs(body)

src/digConfigs.ts

@@ -6,6 +6,7 @@ import { BigipConfObj, TmosApp } from './models'
 import { RegExTree } from './regex';
 import { pathValueFromKey } from './objects';
 import { poolsInPolicy } from './pools';
+import { keyValuePairs } from './deepParse';
 
 
 /**
@@ -45,18 +46,41 @@ export async function digVsConfig(vsName: string, vsConfig: BigipConfObj["ltm"][
         appObj.pool = JSON.parse(JSON.stringify(configTree.ltm?.pool?.[vsConfig.pool]));
         delete appObj.pool.line;
 
-        if(appObj.pool?.members) {
+        if (appObj.pool?.members) {
 
-            Object.keys(appObj.pool?.members).forEach( n => {
+            Object.keys(appObj.pool?.members).forEach(n => {
                 // loop through all the pool members and get the node details
                 const name = n.split(':')[0];
                 const body = configTree.ltm.node[name]
                 if (body) {
                     appObj.lines.push(`ltm node ${name} {${body.line}}`);
                 }
-
+
+            })
+        }
+
+        if (appObj?.pool?.monitor) {
+
+            appObj.pool.monitor.map(x => {
+
+                const p = pathValueFromKey(configTree.ltm?.monitor, x)
+                if (p) {
+
+                    // clone the app config
+                    const tmpObj = JSON.parse(JSON.stringify(p.value));
+                    delete tmpObj.line;
+
+                    // replace the monitor name with the full monitor config object in the app
+                    const idx = appObj.pool.monitor.indexOf(x)
+                    appObj.pool.monitor[idx] = tmpObj;
+
+                    appObj.lines.push(`ltm monitor ${p.path} ${p.key} { ${p.value.line} }`);
+
+                }
             })
         }
+
+        appObj;
     }
 
     if (appObj.profiles) {
@@ -68,19 +92,19 @@ export async function digVsConfig(vsName: string, vsConfig: BigipConfObj["ltm"][
             // check the ltm profiles
             const x = pathValueFromKey(configTree.ltm?.profile, name);
             if (x) {
-                appObj.lines.push(`ltm profile ${x.path} ${x.key} {${x.value}}`);
+                appObj.lines.push(`ltm profile ${x.path} ${x.key} {${x.value.line}}`);
             }
 
             // check apm profiles
             const y = pathValueFromKey(configTree?.apm?.profile?.access, name);
             if (y) {
-                appObj.lines.push(`apm profile access ${y.path} ${y.key} {${y.value}}`);
+                appObj.lines.push(`apm profile access ${y.path} ${y.key} {${y.value.line}}`);
             }
 
             // check asm profile
             const z = pathValueFromKey(configTree?.asm?.policy, name);
             if (z) {
-                appObj.lines.push(`asm policy ${z.path} ${z.key} {${z.value}}`);
+                appObj.lines.push(`asm policy ${z.path} ${z.key} {${z.value.line}}`);
             }
         })
     }
@@ -117,10 +141,10 @@ export async function digVsConfig(vsName: string, vsConfig: BigipConfObj["ltm"][
 
             const x = pathValueFromKey(configTree.ltm?.policy, name)
             if (x) {
-                appObj.lines.push(`ltm policy ${x.key} {${x.value}}`);
+                appObj.lines.push(`ltm policy ${x.key} {${x.value.line}}`);
 
                 // got through each policy and dig references (like pools)
-                const pools = poolsInPolicy(x.value)
+                const pools = poolsInPolicy(x.value.line)
 
                 if (pools) {
                     pools.forEach(pool => {
@@ -142,15 +166,15 @@ export async function digVsConfig(vsName: string, vsConfig: BigipConfObj["ltm"][
         // dig profiles details
         const x = pathValueFromKey(configTree.ltm?.persistence, appObj.persist)
         if (x) {
-            appObj.lines.push(`ltm persistence ${x.path} ${x.key} {${x.value}}`);
+            appObj.lines.push(`ltm persistence ${x.path} ${x.key} {${x.value.line}}`);
         }
     }
 
     if (appObj['fallback-persistence']) {
         // dig profiles details
         const x = pathValueFromKey(configTree.ltm?.persistence, appObj['fallback-persistence'])
         if (x) {
-            appObj.lines.push(`ltm persistence ${x.path} ${x.key} {${x.value}}`);
+            appObj.lines.push(`ltm persistence ${x.path} ${x.key} {${x.value.line}}`);
         }
     }
 

src/ltm.ts

@@ -23,6 +23,7 @@ import { DigGslb } from './digGslb';
 import { parseDeep } from './deepParse';
 // import { XMLParser } from 'fast-xml-parser';
 import { deepmergeInto } from 'deepmerge-ts';
+import { balancedRx1, balancedRxAll } from './tmos2json';
 
 
 
@@ -277,6 +278,33 @@ export default class BigipConfig extends EventEmitter {
 
     async parentTmosObjects(conf: ConfigFile): Promise<string[]> {
 
+        // // *** todo: may need to copy the config body here so we don't modify the object that will get added to the sources
+
+        // const preRx = /(apm|ltm|gtm|asm|security|net|pem|sys|wom|ilx|auth|analytics|wom) [\w\-\/:. ]+ {/
+
+        // let pRx: RegExpMatchArray | null;
+        // const ret: any[] = []
+        // let rest: string;
+
+        // do {
+
+        //     // run the rx to find the beginning of a backeted object
+        //     pRx = conf.content.match(preRx);
+
+        //     if(pRx) {
+        //         // catpure the bracketed object
+        //         const r = balancedRx1(pRx[0], conf.content);
+
+        //         if(r) {
+        //             // push key/value from bracketed object to the return array
+        //             ret.push({key: r.prefaceKey, body: r.body});
+        //             // update the original string so we can dig out the next object
+        //             conf.content = r.rest;
+        //         }
+        //     }
+        // }
+        // while (pRx);
+
         // this is needed to mark the end of the file, and get the regex to complete
         //      the parentObjects rx relies on the start of the next known parent object (ltm|apm|gtm|asm|sys|...)
         const confContent = conf.content.concat('---end---');

src/models.ts

@@ -29,6 +29,7 @@ export type BigipConfObj = {
                 name: string;
                 partition: string;
                 members: { [key: string]: unknown };
+                monitor: string[];
             };
         };
         node?: {
@@ -39,11 +40,30 @@ export type BigipConfObj = {
                 address: string;
             };
         };
-        monitor?: { [key: string]: string };
-        profile?: { [key: string]: string };
-        policy?: { [key: string]: string };
+        monitor?: {
+            [key: string]: {
+                name: string;
+                partition: string;
+                line: string;
+                folder?: string;
+            }
+        };
+        profile?: {
+            [key: string]: {
+                line: string;
+            }
+        };
+        policy?: {
+            [key: string]: {
+                line: string;
+            }
+        };
         rule?: { [key: string]: string };
-        persistence?: { [key: string]: string };
+        persistence?: {
+            [key: string]: {
+                line: string;
+            }
+        };
         snatpool?: {
             [key: string]: {
                 line: string;

src/objects.ts

@@ -6,7 +6,7 @@ import { BigipConfObj } from "./models";
 export type RetObj = {
     path?: string,
     key?: string,
-    value?: string
+    value?: { line: string; };
 }
 
 

src/regex.ts

@@ -86,7 +86,7 @@ export class RegExTree {
             memberDef: /(\/[\w\-\/.]+:\d+) {\s+address(.+?)\s+}/g,
             memberFqdnDef: /(\/[\w\-\/.]+:\d+) {\s+fqdn {\s+([\s\S]+?)\s+}\s+}/g,
             nodesFromMembers: /(\/\w+\/.+?)(?=:)/g,
-            monitors: /monitor ([\/\w.]+?)\n/
+            monitors: /monitor (?:min \d of )?([{}\/\w. ]+)/
         },
         profiles: {
             asmProf: /ASM_([\w-.]+)/,
@@ -213,6 +213,11 @@ export class RegExTree {
      */
     getTMOSversion(config: string): string {
         const version = config.match(this.tmosVersionReg);
+
+        // learning that making sure the tmos version is in every file can be problematic
+        //      gonna lean into the idea that if we detect a version at some point we will use it,
+        //       else default to the current regex tree
+
         if (version) {
             //found tmos version
             // if(version[1] === this.tmosVersion) {
@@ -225,8 +230,8 @@ export class RegExTree {
         } else {
             const msg = 'tmos version not detected -> meaning this probably is not a bigip.conf'
             logger.error(msg)
-            Promise.reject(msg);
-            throw new Error(msg);
+            // Promise.reject(msg);
+            // throw new Error(msg);
         }
 
     }

src/unPackerStream.ts

@@ -173,16 +173,28 @@ export class UnPacker extends EventEmitter {
  */
 export function fileFilter(name: string): boolean {
 
+    /**
+     * 5.24.2023
+     * 
+     * had a problem where there were "backup" config files causing conflicts with the file filter process
+     * 
+     * todo: tighten the file filter based on this article
+     * K26582310: Overview of BIG-IP configuration files
+     * https://my.f5.com/manage/s/article/K26582310
+     * 
+     */
+
+
 
     /**
      * breakind down this bigger regex for explaination
      * added to list of regex's below
      */
-    const allConfs = multilineRegExp([
+    const allPartitionsConfs = multilineRegExp([
         // base /config directory
         /^config/,
-        // optional /partitions directory including /partition name directory
-        /(?:\/partitions\/[\w-]+?)?/,
+        // /partitions directory including /partition name directory
+        /\/partitions\/[\w-]+?/,
         // any bigip*.conf file
         /\/bigip(?:[\w-]*).conf$/
     ], undefined)
@@ -222,7 +234,13 @@ export function fileFilter(name: string): boolean {
      * only one has to pass to return true
      */
     const fileRegexs: RegExp[] = [
-        allConfs,                           // all .conf files (including partitions)
+        allPartitionsConfs,                           // all .conf files (including partitions)
+        /^config\/bigip.conf$/,             // main/base config file
+        /^config\/bigip_gtm.conf$/,             // base gtm config file
+        /^config\/bigip_base.conf$/,        // system/network config file
+        /^config\/bigip_script.conf$/,        // scripts/iApps config file
+        /^config\/bigip_user.conf$/,        // user config file
+        /^config\/user_alert.conf$/,        // user config file
         /^config\/bigip.license$/,          // license file
         /^config\/profile_base.conf$/,      // default profiles
         /^config\/low_profile_base.conf$/,  // default system profiles

tests/040_waf.tests.ts

@@ -72,7 +72,7 @@ describe('WAF/ASM parsing/abstraction', async function () {
 
         for await (const [k, v] of Object.entries(ltps)) {
 
-            const body = v.replace(/\n +/g, ' ')
+            const body = v.line.replace(/\n +/g, ' ')
 
             const iAsm = body.match(device.rx!.ltm.ltPolicies.asmRef)
             if (iAsm) {