Skip to content

Commit

Permalink
Merge pull request #703 from intersystems/v0.10.x-feat-init-cpf-merge
Browse files Browse the repository at this point in the history
Feat: CPF Merge
  • Loading branch information
isc-shuliu authored Jan 29, 2025
2 parents 39502db + e2af5aa commit f2c631f
Show file tree
Hide file tree
Showing 20 changed files with 352 additions and 41 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -190,7 +190,7 @@ jobs:
CONTAINER=$(docker run --network zpm --rm -d ${{ steps.image.outputs.name }} ${{ steps.image.outputs.flags }})
docker cp tests/migration/v0.7-to-v0.9/. $CONTAINER:/tmp/test-package/
sleep 5; docker exec $CONTAINER /usr/irissys/dev/Cloud/ICM/waitISC.sh
docker exec -i $CONTAINER iris session iris -UUSER << EOF
docker exec -i $CONTAINER iris session iris -UUSER << 'EOF'
s version="0.7.4" s r=##class(%Net.HttpRequest).%New(),r.Server="pm.community.intersystems.com",r.SSLConfiguration="ISC.FeatureTracker.SSL.Config" d r.Get("/packages/zpm/"_version_"/installer"),$system.OBJ.LoadStream(r.HttpResponse.Data,"c")
zpm "list":1
zpm "install dsw":1
Expand Down
4 changes: 3 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- #609 Added support for `-export-deps` when running the "Package" phase of lifecycle
- #541 Added support for ORAS repository
- #704 Added support for passing in env files via `-env /path/to/env1.json;/path/to/env2.json` syntax
- #702 Added a new lifecycle phase `Initialize` which is used for preload
- #702 Added a `<CPF/>` resource, which can be used for CPF merge before/after a specified lifecycle phase or in a custom lifecycle phase.
- #716 Added support to publish under external name by passing `-use-external-name` or `-use-ext`.

### Changed
-
- #702 Preload now happens as part of the new `Initialize` lifecycle phase. `zpm "<module> reload -only"` will no longer auto compile resources in `/preload` directory.

### Fixed
- #474: When loading a .tgz/.tar.gz package, automatically locate the top-most module.xml in case there is nested directory structure (e.g., GitHub releases)
Expand Down
7 changes: 7 additions & 0 deletions src/cls/IPM/DataType/CustomPhaseName.cls
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
Class %IPM.DataType.CustomPhaseName Extends %Library.String [ ClassType = datatype ]
{

/// The maximum number of characters the string can contain.
Parameter MAXLEN As INTEGER = 255;

}
14 changes: 14 additions & 0 deletions src/cls/IPM/DataType/PhaseName.cls
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
Class %IPM.DataType.PhaseName Extends %Library.String [ ClassType = datatype ]
{

/// The maximum number of characters the string can contain.
Parameter MAXLEN As INTEGER = 50;

/// Used for enumerated (multiple-choice) attributes.
/// <var>VALUELIST</var> is either a null string ("") or a delimiter
/// separated list (where the delimiter is the first character) of logical values.
/// If a non-null value is present, then the attribute is restricted to values
/// in the list, and the validation code simply checks to see if the value is in the list.
Parameter VALUELIST = ",Clean,Initialize,Reload,*,Validate,ExportData,Compile,Activate,Document,MakeDeployed,Test,Package,Verify,Publish,Configure,Unconfigure";

}
14 changes: 14 additions & 0 deletions src/cls/IPM/DataType/PhaseWhen.cls
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
Class %IPM.DataType.PhaseWhen Extends %Library.String [ ClassType = datatype ]
{

/// The maximum number of characters the string can contain.
Parameter MAXLEN As INTEGER = 50;

/// Used for enumerated (multiple-choice) attributes.
/// <var>VALUELIST</var> is either a null string ("") or a delimiter
/// separated list (where the delimiter is the first character) of logical values.
/// If a non-null value is present, then the attribute is restricted to values
/// in the list, and the validation code simply checks to see if the value is in the list.
Parameter VALUELIST = ",Before,After";

}
24 changes: 24 additions & 0 deletions src/cls/IPM/DataType/ResourceDirectory.cls
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
Class %IPM.DataType.ResourceDirectory Extends %Library.String [ ClassType = datatype ]
{

Parameter MAXLEN = 255;

/// Tests if the logical value <var>%val</var>, which is a string, is valid.
/// The validation is based on the class parameter settings used for the class attribute this data type is associated with.
/// In this case, <a href="#MINLEN">MINLEN</a>, <a href="#MAXLEN">MAXLEN</a>, <a href="#VALUELIST">VALUELIST</a>, and <a href="#PATTERN">PATTERN</a>.
ClassMethod IsValid(%val As %RawString) As %Status [ ServerOnly = 0 ]
{
If $Extract(%val) = "/" {
Return $$$ERROR($$$GeneralError, "Resource directory cannot start with a slash.")
}
Set segments = $ListFromString(%val, "/")
Set ptr = 0
While $ListNext(segments, ptr, seg) {
If seg = ".." {
Return $$$ERROR($$$GeneralError, "For security reasons, resource directory cannot contain '..'.")
}
}
Return $$$OK
}

}
63 changes: 42 additions & 21 deletions src/cls/IPM/Lifecycle/Base.cls
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ Property PhaseList As %List;

/// $ListBuild list of phases in this lifecycle. <br />
/// For each phase name, an instance method named "%<phase name>" must be defined in the class with a return type of %Status.
Parameter PHASES = {$ListBuild("Clean","Reload","*","Validate","ExportData","Compile","Activate","Document","MakeDeployed","Test","Package","Verify", "Publish", "Configure","Unconfigure")};
Parameter PHASES = {$ListBuild("Clean","Initialize", "Reload","*","Validate","ExportData","Compile","Activate","Document","MakeDeployed","Test","Package","Verify", "Publish", "Configure","Unconfigure")};

Property Payload As %Stream.Object [ Private ];

Expand Down Expand Up @@ -121,7 +121,7 @@ ClassMethod GetDefaultResourceProcessorProc(pLifecycleClass As %Dictionary.Class
ClassMethod GetCompletePhases(pPhases As %List) As %List
{
// If there is only phase, and the phase is not found in standard phases, it is a custom phase. Return as-is.
If ($ListLength(pPhases) = 1) && ('$ListFind(..#PHASES, $List(pPhases,1))) {
If ($ListLength(pPhases) = 1) && ('$ListFind(..#PHASES, ..MatchSinglePhase($List(pPhases,1)))) {
Return pPhases
}
For i=1:1:$ListLength(pPhases) {
Expand All @@ -142,18 +142,19 @@ ClassMethod GetCompletePhasesForOne(pOnePhase As %String) As %List

Quit $Case(pOnePhase,
"clean": $ListBuild("Clean"),
"reload": $ListBuild("Reload","*"),
"validate": $ListBuild("Reload","*","Validate"),
"initialize": $ListBuild("Initialize"),
"reload": $ListBuild("Initialize","Reload","*"),
"validate": $ListBuild("Initialize","Reload","*","Validate"),
"exportdata": $ListBuild("ExportData"),
"compile": $ListBuild("Reload","*","Validate","Compile"),
"activate": $ListBuild("Reload","*","Validate","Compile","Activate"),
"compile": $ListBuild("Initialize","Reload","*","Validate","Compile"),
"activate": $ListBuild("Initialize","Reload","*","Validate","Compile","Activate"),
"document": $ListBuild("Document"),
"makedeployed": $ListBuild("MakeDeployed"),
"test": $ListBuild("Reload","*","Validate","Compile","Activate","Test"),
"package": $ListBuild("Reload","*","Validate","Compile","Activate","Package"),
"verify": $ListBuild("Reload","*","Validate","Compile","Activate","Package","Verify"),
"register": $ListBuild("Reload","*","Validate","Compile","Activate","Package","Register"),
"publish": $ListBuild("Reload","*","Validate","Compile","Activate","Package","Register","Publish"),
"test": $ListBuild("Initialize","Reload","*","Validate","Compile","Activate","Test"),
"package": $ListBuild("Initialize","Reload","*","Validate","Compile","Activate","Package"),
"verify": $ListBuild("Initialize","Reload","*","Validate","Compile","Activate","Package","Verify"),
"register": $ListBuild("Initialize","Reload","*","Validate","Compile","Activate","Package","Register"),
"publish": $ListBuild("Initialize","Reload","*","Validate","Compile","Activate","Package","Register","Publish"),
"configure": $ListBuild("Configure"),
"unconfigure": $ListBuild("Unconfigure"),
: ""
Expand All @@ -166,6 +167,7 @@ ClassMethod MatchSinglePhase(pOnePhase As %String) As %String
Set phase = $ZCONVERT(pOnePhase, "L")
Quit $Case(phase,
"clean": "Clean",
"initialize": "Initialize",
"reload": "Reload",
"validate": "Validate",
"exportdata": "ExportData",
Expand Down Expand Up @@ -505,6 +507,35 @@ Method %Unconfigure(ByRef pParams) As %Status
Quit tSC
}

Method %Initialize(ByRef pParams) As %Status
{
Set status = $$$OK
Try {
Set key = ""
For {
Set resource = ..Module.Resources.GetNext(.key)
Quit:key=""
If $IsObject(resource.Processor) {
Set status = $$$ADDSC(status,resource.Processor.OnPhase("Initialize",.pParams))
}
}

Set preloadRoot = $Get(pParams("RootDirectory"))_"preload"
Set verbose = $Get(pParams("Verbose"))
If ##class(%File).DirectoryExists(preloadRoot) {
Set tSC = $System.OBJ.ImportDir(preloadRoot, "*", $Select(verbose:"d",1:"-d")_$Select($Tlevel:"/multicompile=0", 1: ""), , 1, .tImported)
If $$$ISERR(tSC) { Quit }
Set tSC = ##class(%IPM.Utils.LegacyCompat).UpdateSuperclassAndCompile(.tImported)
If $$$ISERR(tSC) { Quit }
} ElseIf verbose {
Write !,"Skipping preload - directory does not exist."
}
} Catch ex {
Set status = ex.AsStatus()
}
Quit status
}

Method %Reload(ByRef pParams) As %Status
{
Set tSC = $$$OK
Expand Down Expand Up @@ -581,16 +612,6 @@ Method %Reload(ByRef pParams) As %Status

$$$ThrowOnError(..InstallPythonRequirements(tRoot, .pParams))

Set tPreloadRoot = tRoot_"preload"
If ##class(%File).DirectoryExists(tPreloadRoot) {
Set tSC = $System.OBJ.ImportDir(tPreloadRoot, "*", $Select(tVerbose:"d",1:"-d")_$Select($Tlevel:"/multicompile=0", 1: ""), , 1, .tImported)
If $$$ISERR(tSC) { Quit }
Set tSC = ##class(%IPM.Utils.LegacyCompat).UpdateSuperclassAndCompile(.tImported)
If $$$ISERR(tSC) { Quit }
} ElseIf tVerbose {
Write !,"Skipping preload - directory does not exist."
}

// Reload the module definition
Set tSC = ..Module.%Reload()
If $$$ISERR(tSC) {
Expand Down
1 change: 0 additions & 1 deletion src/cls/IPM/ResourceProcessor/Abstract.cls
Original file line number Diff line number Diff line change
Expand Up @@ -231,7 +231,6 @@ Method GetSourceControlInfo(Output pInfo As %IPM.ExtensionBase.SourceControl.Res
Quit $$$OK
}


/// Evaluates an expression in a provided string. <br />
/// These special expressions are case-insensitive. <br />
/// Current valid expressions:
Expand Down
93 changes: 93 additions & 0 deletions src/cls/IPM/ResourceProcessor/CPF.cls
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
Include %IPM.Formatting

Class %IPM.ResourceProcessor.CPF Extends (%IPM.ResourceProcessor.Abstract, %IPM.ResourceProcessor.CustomPhaseMixin)
{

/// Comma-separated list of resource attribute names that this processor uses
Parameter ATTRIBUTES As STRING = "Name,Directory,Phase,CustomPhase,When";

/// Description of resource processor class (shown in UI)
Parameter DESCRIPTION As STRING = "Merges the specified CPF file in the specified lifecycle phase (""Initialize"" by default).";

/// Directory containing the CPF file to merge
Property Directory As %IPM.DataType.ResourceDirectory [ InitialExpression = "cpf" ];

/// FileN ame of the CPF merge file
Property Name As %IPM.DataType.ResourceName [ Required ];

/// The phase before/after which the CPF file should be merged
Property Phase As %IPM.DataType.PhaseName [ InitialExpression = "Initialize" ];

/// When to merge the CPF file: Before or After the specified phase. This only applies to the standard phases.
Property When As %IPM.DataType.PhaseWhen [ InitialExpression = "Before" ];

Method OnBeforePhase(pPhase As %String, ByRef pParams) As %Status
{
If (..When = "Before") && (..Phase = pPhase) && (..CustomPhase = "") {
Quit ..DoMerge(.pParams)
}
Quit $$$OK
}

Method OnAfterPhase(pPhase As %String, ByRef pParams) As %Status
{
If (..When = "After") && (..Phase = pPhase) && (..CustomPhase = "") {
Quit ..DoMerge(.pParams)
}
Quit $$$OK
}

Method OnCustomPhase(pCustomPhase As %String, ByRef pParams) As %Status
{
If (..CustomPhase = pCustomPhase) {
Quit ..DoMerge(.pParams)
}
Quit $$$OK
}

Method DoMerge(ByRef pParams) As %Status
{
Try {
Set verbose = $GET(pParams("Verbose"))
Set root = ..ResourceReference.Module.Root
Set sourcesRoot = ..ResourceReference.Module.SourcesRoot
// Use Construct first, rather than NormalizeFilename, so we don't have to deal with leading/trailing slashes
Set dir = $SELECT($$$isWINDOWS: $REPLACE(..Directory, "/", "\"), 1: ..Directory)
Set dir = ##class(%File).Construct(root, sourcesRoot, dir)
Set filename = ##class(%File).NormalizeFilename(..Name, dir)
If (filename = "") || ('##class(%File).Exists(filename)) {
$$$ThrowStatus($$$ERROR($$$GeneralError, $$$FormatText("CPF file '%1' not found in directory '%2'", ..Name, dir)))
}

Set stream = ##class(%Stream.FileCharacter).%New()
$$$ThrowOnError(stream.LinkToFile(filename))
If verbose {
Write !, "Merging CPF file: ", filename, !
Do stream.OutputToDevice()
}
Do ..MergeCPF(filename)
} Catch ex {
Return ex.AsStatus()
}
Return $$$OK
}

ClassMethod MergeCPF(file As %String)
{
// TODO The $zf(-100) callout is much slower than ##class(Config.CPF).Merge()
// Figure out why ##class(Config.CPF).Merge() doesn't work
// c.f. https://github.com/intersystems/ipm/pull/703#discussion_r1917290136

Set args($INCREMENT(args)) = "merge"
Set args($INCREMENT(args)) = ##class(%SYS.System).GetInstanceName()
Set args($INCREMENT(args)) = file

// Somehow, if the STDOUT is not set, the merge will silently fail
Set flags = "/SHELL/LOGCMD/STDOUT=""zf100stdout""/STDERR=""zf100stderr"""
Set status = $ZF(-100, flags, "iris", .args)
If status '= 0 {
$$$ThrowStatus($$$ERROR($$$GeneralError, "Error merging CPF file. $zf(-100) exited with "_status))
}
}

}
14 changes: 14 additions & 0 deletions src/cls/IPM/ResourceProcessor/CustomPhaseMixin.cls
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
/// Some processors may need to do something custom when a custom phase is executed.
/// Such as the CPF processor, which optionally merges a CPF file during a custom phase.
/// This mixin provides the CustomPhase property and the OnCustomPhase method.
Class %IPM.ResourceProcessor.CustomPhaseMixin
{

Property CustomPhase As %IPM.DataType.CustomPhaseName;

Method OnCustomPhase(pCustomPhase As %String, ByRef pParams) As %Status
{
Quit $$$OK
}

}
6 changes: 3 additions & 3 deletions src/cls/IPM/Storage/InvokeReference.cls
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,12 @@ Property Class As %String(MAXLEN = 255, XMLPROJECTION = "ATTRIBUTE") [ Required

Property Method As %String(MAXLEN = 255, XMLPROJECTION = "ATTRIBUTE") [ Required ];

Property Phase As %String(MAXLEN = 255, XMLPROJECTION = "ATTRIBUTE") [ InitialExpression = "Configure" ];
Property Phase As %IPM.DataType.PhaseName(XMLPROJECTION = "ATTRIBUTE") [ InitialExpression = "Configure" ];

/// If provided, the Phase property will be ignored. This CustomPhase will be used and no corresponding lifecycle is required.
Property CustomPhase As %String(MAXLEN = 255, XMLPROJECTION = "ATTRIBUTE");
Property CustomPhase As %IPM.DataType.CustomPhaseName(XMLPROJECTION = "ATTRIBUTE");

Property When As %String(MAXLEN = 255, VALUELIST = ",Before,After", XMLPROJECTION = "ATTRIBUTE") [ InitialExpression = "After", SqlFieldName = _WHEN ];
Property When As %IPM.DataType.PhaseWhen(XMLPROJECTION = "ATTRIBUTE") [ InitialExpression = "After", SqlFieldName = _WHEN ];

Property CheckStatus As %Boolean(XMLPROJECTION = "ATTRIBUTE") [ InitialExpression = 0 ];

Expand Down
34 changes: 31 additions & 3 deletions src/cls/IPM/Storage/Module.cls
Original file line number Diff line number Diff line change
Expand Up @@ -177,11 +177,27 @@ Method GetCustomPhases(Output pPhases)
Set key = ""
For {
Set tInvoke = ..Invokes.GetNext(.key)
Quit:(key = "")
If key = "" {
Quit
}
If (tInvoke.CustomPhase '= "") {
Set pPhases($$$lcase(tInvoke.CustomPhase)) = tInvoke.CustomPhase
}
}
For {
Set tResource = ..Resources.GetNext(.key)
If key = "" {
Quit
}
Set tProcessor = tResource.Processor
// %IsA() only returns true if the class is the "primary" superclass, while %Extends() works for other superclasses (such as mixins).
If $IsObject(tProcessor) && tProcessor.%Extends("%IPM.ResourceProcessor.CustomPhaseMixin") {
Set cp = tProcessor.CustomPhase
If cp '= "" {
Set pPhases($$$lcase(cp)) = cp
}
}
}
}

/// Execute multiple lifecycle phases in sequence. Execution is terminated if one fails.
Expand Down Expand Up @@ -350,7 +366,20 @@ ClassMethod ExecutePhases(pModuleName As %String, pPhases As %List, pIsComplete
}
Quit:$$$ISERR(tSC)

If 'tIsCustomPhase {
If tIsCustomPhase {
Set tKey = ""
For {
Set tResource = tModule.Resources.GetNext(.tKey)
If tKey = "" {
Quit
}
Set tProcessor = tResource.Processor
// %IsA() only returns true if the class is the "primary" superclass, while %Extends() works for other superclasses (such as mixins).
If $IsObject(tProcessor) && tProcessor.%Extends("%IPM.ResourceProcessor.CustomPhaseMixin") {
$$$ThrowOnError(tProcessor.OnCustomPhase(tOnePhase, .pParams))
}
}
} Else {
// Lifecycle before / (phase) / after
$$$ThrowOnError(tLifecycle.OnBeforePhase(tOnePhase,.pParams))
$$$ThrowOnError($Method(tLifecycle,"%"_tOnePhase,.pParams))
Expand Down Expand Up @@ -1612,4 +1641,3 @@ Storage Default
}

}

Loading

0 comments on commit f2c631f

Please sign in to comment.