Skip to main content

Using NQE to do Configurations Compliance checks require having the proper structure for the searched text pattern.

Ex: this will not match eventhough that line exists within the configurations, but it’s not in the root level

PATTERN=```
set admintimeout 10
```;

So, to match this line we have to provide the proper structure, or recusively add an extra level until a match is found

So, modifing that example to one of the below patterns should work

PATTERN1=```
config system global
set admintimeout 10
```;

PATTERN2=```
{(string)+}
set admintimeout 10
```;

So, in this example it was only 1 level down, so it’s straight forward to fix.

But having to work with different patterns with different nesting levels the below code can be used to automatically modify the provided pattern and recursively search all the different levels

 

Beware that there are many limitations due to the actual version of the language.

ex: splitting a string is not straight forward, so the pattern has to be provided as a list of strings

ex:

PATTERN=T"config system global"," set admintimeout 10"];

The other limitation, is having this in a different file would force yout to use a PatternBlocks<{}> with no variables. So, it’s better to copy this code within the same nqe script you’re working on if this is needed

 

The below code block is a PoC for that implementation

spaces(n)=
foreach x in fromTo(0, n)
// don't prepend white spaces in the 1st line
where x > 0
select " ";

new_headers(n)=
foreach x in fromTo(0, n)
where x > 0
let pre = join("",spaces(x))
select pre + "{a" + toString(x-1) + ":(string)+}";

mod_cmd(cmd,n)=join("", spaces(n))+cmd;
mod_cmds(cmds,n)=(foreach cmd in cmds select(mod_cmd(cmd, n)));

export gen_pattern(str_list: List<String>, depth:Number)=blockPattern(join("\n", new_headers(depth) + mod_cmds(str_list, depth+1)));

export gen_patterns(str_list: List<String>, depth:Number)=
foreach x in fromTo(0, depth)
select gen_pattern(str_list, depth);

 

And a full implementation would look like this

 

//
// recursive pattern
//

spaces(n)=
foreach x in fromTo(0, n)
// don't prepend white spaces in the 1st line
where x > 0
select " ";

new_headers(n)=
foreach x in fromTo(0, n)
where x > 0
let pre = join("",spaces(x))
select pre + "{a" + toString(x-1) + ":(string)+}";

mod_cmd(cmd,n)=join("", spaces(n))+cmd;
mod_cmds(cmds,n)=(foreach cmd in cmds select(mod_cmd(cmd, n)));

export gen_pattern(str_list: List<String>, depth:Number)=blockPattern(join("\n", new_headers(depth) + mod_cmds(str_list, depth+1)));

export gen_patterns(str_list: List<String>, depth:Number)=
foreach x in fromTo(0, depth)
select gen_pattern(str_list, depth);

//
// Helper functions:
//

export parse_config_commands(device:Device, pattern:PatternBlocks<{}>, all_cmd_types:Bool, cmd_type:CommandType, cmd_text:String)=
foreach command in device.outputs.commands
where all_cmd_types || command.commandType == cmd_type
let continue = if isPresent(cmd_text) && length(cmd_text)>0
then isPresent(command.commandText) && matches(command.commandText,cmd_text)
else true
where continue
let config = parseConfigBlocks(device.platform.os, command.response)
let results= blockMatches(config, pattern)
foreach res in results
select {command_type: command.commandType, matches: res};


export gen_pattern_then_parse(device:Device, str_list: List<String>, depth:Number, all_cmd_types:Bool, cmd_type:CommandType, cmd_text:String)=
parse_config_commands(device, gen_pattern(str_list, depth), all_cmd_types, cmd_type, cmd_text);

longest_matches(l)=length(l);

export gen_patterns_then_parse(device:Device, str_list: List<String>, depth:Number, all_cmd_types:Bool, cmd_type:CommandType, cmd_text:String)=
maxBy(foreach n in fromTo(0, depth)
select gen_pattern_then_parse(device, str_list, n, all_cmd_types, cmd_type, cmd_text)
, longest_matches);

 

 

Using that code:

ADMIN_TIMEOUT=A"set admintimeout 10"];
IDLE_TIMEOUT=>"set idle_timeout 10"];

foreach device in network.devices
let admin_timeout = gen_patterns_then_parse(device, ADMIN_TIMEOUT, 2, false, CommandType.CUSTOM, "show full-configuration")
let idle_timeout = gen_pattern_then_parse(device, IDLE_TIMEOUT, 1, false, CommandType.CUSTOM, null:String)

select {
device: device.name,
admin_timeout,
idle_timeout
}

 

Sample output

 

I hope this code helps

Also, improvements to that is v. welcome as well!

Awesome work @Ahmed , As with other languages there are many ways to accomplish the same task with NQE. There is lots to unpack in your example, you really have demonstrated advanced skills in leveraging NQE.

If the main objective was extracting specific patterns out of configurations using blockPatterns, you can definitely assign your patternBlocks to variables. We do this all the time to build up baseline configurations and differential analysis. 

Note the trick here is that the union-type all_patterns needs to carry all the same types so this might need to be broken up based on your objective.

Note: It is often useful when sharing with the community to add your test patterns into your examples, this way others who may not have the same data as you can understand what the query does by executing on their own instance of Forward Enterprise. 

Again.. Awesome post. 

 

test1 =
"""
config system global
set admintimeout 10
set idle_timeout 2
set https_port 444
""";

admin_timeout = ```
config system global
set admintimeout {arg:number}
```;

idle_timeout = ```
config system global
set idle_timeout {arg:number}
```;

https_port = ```
config system global
set https_port {arg:number}
```;

all_patterns =
{ selector: "admin_timeout", pattern: admin_timeout },
{ selector: "idle_timeout", pattern: idle_timeout },
{ selector: "https_port", pattern: https_port }
];

foreach x in x0]
foreach p in all_patterns
let blocks = parseConfigBlocks(OS.UNKNOWN, test1)
let matches = blockMatches(blocks, p.pattern)
foreach match in matches
select { selector: p.selector, value: match.data.arg, pattern: p.pattern }

 


Hi Gary,

 

Thanks a lot for your nice  feedback

Yeah the included example is a simple one. but this could be used within more complex patterns that are buried deeper within the configurations tree.

Sure, I tried many ways to get around that issue while passing a generic match with different capture variables with no success due to having the code placed in a shared library (diffferent NQE file.)

the same code works seemlessly with different capture variables within the same NQE Script.

 

Will try to integrate your example && update the post/keep you aligned.

 

Thanks


@Ahmed Looking forward to it. Can you provide some examples “ more complex patterns that are buried deeper within the configurations tree.” 

We have many methods that might help.


Sure,

This is an exmaple of that

config
devices
localhost.localdomain
vsys
{vsysName: string}
rulebase
security
rules
{ruleName:string}
rule-type {ruletype:string}
description {description: string}
source {srcIps: string}
from {fromZone:string}
to {toZone:string}
destination {dstIps:string}
application {app:string}
service {service:string}
action {action:string}

 

The other solid use case for this pattern is when you’re not actually sure where is it in the tree.

This code will allow you to find the whole tree path, down to your match.

 

I’m sure you have other internal tools to do this, would be great if we can have access to :)

 


You can see that I used a more “brute force” approach in this query that is in the forward library.

Notice that I created separate patterns for each “depth” into the config.  Then add them together at the end.

/**
* @intent Finds references to undefined prefix lists.
* @description It is best for each prefix list usage to reference a defined prefix list.
* If a prefix list references an undefined prefix list,
* then some default behavior is applied.
* This is likely to be confusing and is likely a mistake.
* This query has been modified to also support the ASA syntax of prefix-lists.
*/
asaPatternDefinition =
```
prefix-list {name:(!"seq" !"in" !"out" !"ip" !"install" !"filter" !"ipv4" string)}
```;
patternDefinition =
```
{"ip" | "ipv6"} prefix-list {name:(!"seq" !"in" !"out" !"ip" !"install" !"filter" !"ipv4" string)}
```;
// Exclude "description" since the text that follows could include "prefix-list".
// Exlucde `neighbor {string} capability orf prefix-list` since the next word is not a reference to a prefix-list.
// Reference: https://www.cisco.com/c/en/us/td/docs/ios-xml/ios/iproute_bgp/configuration/15-mt/irg-15-mt-book/bgp_prefix-based_outbound_route_filtering.pdf
patternLevel0 =
```
{!"description"} {!"action"} {!"rule"} {!("neighbor" string "capability" "orf" "prefix-list")} {!"ip prefix-list"} {!"ipv6 prefix-list"} {(!"prefix-list" string)*} prefix-list {name:((!"seq" !"in" !"out" !"ip" !"install" !"filter" !"ipv4" !"override" string)*)}
```;
patternLevel1 =
```
{string}
{!"description"} {!"action"} {!"rule"} {!("neighbor" string "capability" "orf" "prefix-list")} {(!"prefix-list" string)*} prefix-list {name:((!"seq" !"in" !"out" !"ip" !"install" !"filter" !"ipv4" !"override" string)*)}
```;
// Exclude bgp neighbor capability command
// Exclude action under event manager
// Exclude descriptions that contain prefix-list names
// under roles there were rules that need to be excluded
patternLevel2 =
```
{string}
{string}
{!"description"} {!"action"} {!"rule"} {!("neighbor" string "capability" "orf" "prefix-list")} {(!"prefix-list" string)*} prefix-list {name:((!"seq" !"in" !"out" !"ip" !"install" !"filter" !"ipv4" !"override" string)*)}
```;
patternLevel3 =
```
{string}
{string}
{string}
{!"description"} {!"action"} {!"rule"} {!("neighbor" string "capability" "orf" "prefix-list")} {(!"prefix-list" string)*} prefix-list {name:((!"seq" !"in" !"out" !"ip" !"install" !"filter" !"ipv4" !"override" string)*)}
```;
patternLevel4 =
```
{string}
{string}
{string}
{string}
{!"description"} {!"action"} {!"rule"} {!("neighbor" string "capability" "orf" "prefix-list")} {(!"prefix-list" string)*} prefix-list prefix-list {name:((!"seq" !"in" !"out" !"ip" !"install" !"filter" !"ipv4" !"override" string)*)}
```;
getPrefixListNames(config, pattern) =
foreach match in blockMatches(config, pattern)
select match.data.name;
getASAPrefixListUsages(config) =
foreach pattern
in /patternLevel1, patternLevel2, patternLevel3, patternLevel4]
foreach name in getPrefixListNames(config, pattern)
foreach unpack in name
select distinct unpack;
getPrefixListUsages(config) =
foreach pattern
in /patternLevel0,
patternLevel1,
patternLevel2,
patternLevel3,
patternLevel4
]
foreach name in getPrefixListNames(config, pattern)
foreach unpack in name
select distinct unpack;
foreach device in network.devices
where isPresent(device.platform.model)
where device.platform.vendor in eVendor.CISCO, Vendor.ARISTA]
let config = device.files.config
let prefixListDefinitions = if device.platform.os == OS.ASA
then distinct(getPrefixListNames(config, asaPatternDefinition))
else distinct(getPrefixListNames(config, patternDefinition))
let prefixListUsages = if device.platform.os == OS.ASA
then getASAPrefixListUsages(config)
else getPrefixListUsages(config)
let undefinedPrefixListUsages = prefixListUsages - prefixListDefinitions
let unusedPrefixListDefinitions = prefixListDefinitions - prefixListUsages
select {
violation: length(undefinedPrefixListUsages) > 0,
Device: device.name,
"Count of Undefined prefix-list Usages": length(undefinedPrefixListUsages),
"Undefined prefix-list Usages": undefinedPrefixListUsages,
"Count of Unused prefix-list Definitions": length(unusedPrefixListDefinitions),
"Unused prefix-list Definitions": unusedPrefixListDefinitions,
Vendor: device.platform.vendor,
OS: device.platform.os
}

 


Reply