Creating dynamic records. Additional documentation required

[vitoyucepi]

Question

How can I create 2.0 record with dynamic properties and convert it to json?

Context

General

I want to print information about multiple sources in json format in telnet command response. I expect result to be
validated by schema:

{
  "$id": "sourceInfoSchema.json",
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "type": "object",
  "properties": {
    "success": {
      "type": "boolean"
    }
  },
  "additionalProperties": {
    "$ref": "#/definitions/sourceInfo"
  },
  "definitions": {
    "sourceInfo": {
      "type": "object",
      "properties": {
        "name": {
          "type": "string"
        },
        "isFallible": {
          "type": "boolean"
        },
        "isReady": {
          "type": "boolean"
        },
        "remainingTime": {
          "oneOf": [
            {
              "type": "number"
            },
            {
              "type": "null"
            }
          ]
        },
        "clockTime": {
          "type": "number"
        }
      }
    }
  }
}

For example this is valid response

{
  "success": true,
  "fallback_9550": {
    "name": "Full",
    "isFallible": false,
    "isReady": true,
    "remainingTime": null,
    "clockTime": 570.96
  }
}

1.4.4

In 1.4 there were no records. Also, there's no way to generate objects with different types(not sure). So I can return
only objects like this:

{
  "fallback_9550": {
    "name": "Full",
    "isFallible": "false",
    "isReady": "true",
    "remainingTime": "inf",
    "clockTime": "570.96"
  }
}

I can't set property success in root of json object. All of sourceInfo properties must be of type string. The
schema for this response:

{
  "$id": "sourceInfoSchema_1.4.json",
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "type": "object",
  "additionalProperties": {
    "$ref": "#/definitions/sourceInfo"
  },
  "definitions": {
    "sourceInfo": {
      "type": "object",
      "properties": {
        "name": {
          "type": "string"
        },
        "isFallible": {
          "enum": ["true", "false"]
        },
        "isReady": {
          "enum": ["true", "false"]
        },
        "remainingTime": {
          "$ref": "#/definitions/liquidSoapFloat"
        },
        "clockTime": {
          "$ref": "#/definitions/liquidSoapFloat"
        }
      }
    },
    "liquidSoapFloat": {
      "oneOf": [
        {
          "type": "string",
          "pattern": "^\\d+\\.\\d+$"
        },
        {
          "const": "inf"
        }
      ]
    }
  }
}

I have created a functions to create this json

def formatSourceInfoString(acc, sourcePair)
  let (source, name) = sourcePair
  item = (source.id(source), [
    ("name", "#{name}"),
    ("isFallible", "#{source.fallible(source)}"),
    ("isReady", "#{source.is_ready(source)}"),
    ("remainingTime", "#{source.remaining(source)}"),
    ("clockTime", "#{source.time(source)}"),
  ])
  list.add(item, acc)
end

def getSourceInfo(sourcePairs)
  json_of(compact = true, list.fold(formatSourceInfoString, [], sourcePairs))
end

And call the function like this

schedule = single("schedule.mp3")
radio = single("radio.mp3")
full = single("full.mp3")
getSourceInfo([(schedule, "Schedule"), (radio, "Radio"), (full, "Full")])

2.0.0-beta.3

In 2.0.0 it's possible to create records, that look like json. But there's no documentation how can iterate over record
properties. I want to call function like this:

getSourceInfo({Full = full, Radio = radio, Schedule = schedule})

Iterate over record (expected)

def formatSourceInfoString(infoRecord, sourcesRecordItem)
  let (key, value) = sourceRecordItem
  infoRecordItem = {}
  
  sourceId = source.id(value)
  
  infoRecordItem.name = key
  infoRecordItem.isFallible = source.fallible(value)
  infoRecordItem.isReady = source.is_ready(value)
  
  remainingTime = source.remaining(value)
  if remainingTime = infinity then
    infoRecordItem.remainingTime = null()
  else
    infoRecordItem.remainingTime = remainingTime
  end
  
  clockTime = source.time(value)
  if clockTime = infinity then
    infoRecordItem.clockTime = null()
  else
    infoRecordItem.clockTime = clockTime
  end
  
  infoRecord[sourceId] = recordItem
end

def getSourceInfo(sourceRecord)
  infoRecord = {}
  try
    infoRecord = list.fold(formatSourceInfoString, (), sourcesRecord)
    infoRecord.success = true
  catch _ do
    infoRecord.success = false
  end
  json_of(compact = true, infoRecord)
end

But this will not work because I can't reduce over records.

ERROR: (Failure "TODO: JSON of method not yet implemented")

def getSourceInfo(sourcesRecord)
  try
    json_of(compact = true, {success = true, a = sourcesRecord})
  catch _ do
    json_of(compact = true, {success = false})
  end
end

I want to capture an error.

Function renamed by new variable

def getSourceInfo(pair)
  let (source, name) = pair
  print(source.id(source))
end

getSourceInfo((schedule, "Schedule"))

In 2.0.0 this will not work, because source has no built-in method id.

Record properties in loop

def getSourceInfo(pairs)
  let infoRecord = ()
  for pair = list.iterator(pairs) do
    let infoRecord.a = "1"
  end
  let infoRecord.success = true
  json_of(compact = true, infoRecord)
end

getSourceInfo([(schedule, "Schedule"), (radio, "Radio"), (full, "Full")])

Anything set inside the loop will remain inside the loop

Dereference new record property from variable

def getSourceInfo(pairs)
  let infoRecord = ()
  for pair = list.iterator(pairs) do
    let (src, name) = pair
    let infoRecord[source.id(src)] = name
  end
  let infoRecord.success = true
  json_of(compact = true, infoRecord)
end

getSourceInfo([(schedule, "Schedule"), (radio, "Radio"), (full, "Full")])

Didn't find how to do this in documentation and tests

Can't change set record properties in list.fold

def formatSourceInfoString(accumulator, pair)
  let accumulator.a = "1"
  accumulator
end

def getSourceInfo(pairs)
  infoRecord = list.fold(formatSourceInfoString, (), pairs)
  let infoRecord.success = true
  json_of(compact = true, infoRecord)
end

getSourceInfo([(schedule, "Schedule"), (radio, "Radio"), (full, "Full")])

I can't change accumulator inside function because it'll break types.

[toots]

Hi!

For advanced use of JSON like this I would recommend 2.0.0 indeed. I think your assessment about not being able to create JSON objects of multiple types in 1.4.4 is indeed correct.

Iterate over record (expected)

The function formatSourceInfoString cannot work as-is because records are statically typed so, we need to know all the types in advance, including all key names. Thus, the sourceId key cannot be dynamic.

However, you can also create JSON objects using associative lists, in which case, keys can be dynamic:

# json_of([("foo", 1), ("bar", 3)]);;
- : string = "{ \"foo\": 1, \"bar\": 3 }"

Here's a fixed version of your function:

def formatSourceInfoString(infoRecord, sourceRecordItem)
  let (key, s) = sourceRecordItem
  sourceId = s.id()

  remainingTime = s.remaining()
  remainingTime = (remainingTime == infinity) ? null() : null(remainingTime)

  clockTime = s.time()
  clockTime = (clockTime == infinity) ? null() : null(clockTime)

  recordItem = {
    name          = key,
    isFallible    = s.fallible(),
    isReady       = s.is_ready(),
    remainingTime = remainingTime,
    clockTime     = clockTime
  }

  [(sourceId, recordItem), ...infoRecord]
end

Then you can use a list.fold to create the full associative list.

I have used a couple of tricks, in particular the use of null() in both branches of the if (ternary) statement to avoid the type inference issue you reported in #1816. Also, the parenthesis should go away: #1820

ERROR: (Failure "TODO: JSON of method not yet implemented")

You are right that the failure should be catchable. This is fixed in 09beaad. However, the code needs to be adapted to the functional style of liquidsoap scripts. This should work:

def getSourceInfo(sourceRecord)
  infoRecord =
    try
      { a = null(list.fold(formatSourceInfoString, [], sourceRecord)),
        success = true }
    catch _ do
      { a = null(), success = false }
    end

  json_of(compact = true, infoRecord)
end

Function renamed by new variable

This operator exists in 2.0.0: https://www.liquidsoap.info/doc-dev/reference.html#source.id However, you are naming one of your variable source, which overrides the default source module within the scope of the function

Record properties in loop

Yes, this is the same issue with static typing. Again, I would recommend using an associative list and list.fold for your usage.

Dereference new record property from variable

I don't know if this is possible at the moment. You probably would need to redefine a new record with all the keys that you want to list explicitly listed. Otherwise, we wouldn't be able to type the function.

Here, too, switching to associative lists will solve your issue.

Can't change set record properties in list.fold

Same suggestion re: associative list.

Ultimately, I would say that the only missing spot at the moment is the ability to generate a json object using multiple sources, for instance an associative list (object with dynamic keys but same type for all values) and another module (static keys but multiple possible types for the values). I've captured that in this ticket: #1821

[toots]

Just created a PR to add support for generic abstract json objects: #1824

[vitoyucepi]

My script

def generateSourceInfoRecord(resultRecord, sourceNamePair)
	  let (src, name) = sourceNamePair
	  
	  recordItem = ()
	  
	  sourceId = source.id(src)
	  
	  let recordItem.namespace = name
	  let recordItem.isFallible = source.fallible(src)
	  let recordItem.isReady = source.is_ready(src)
	  let recordItem.clockTime = source.time(src)
	  
	  remainingTime = source.remaining(src)
	  let recordItem.remainingTime = (remainingTime != infinity) ? remainingTime : null()
	  
	  [(sourceId, recordItem), ...resultRecord]
end

def generateResponse(sourceNamePairs)
	  resultRecord = list.fold(generateSourceInfoRecord, [], sourceNamePairs)
	  # let resultRecord.success = true
	  json_of(compact = true, resultRecord)
end

generateResponse([
  (radio, "Radio"),
  (full, "Full"),
  (schedule, "Schedule"),
  (security, "Security"),
  (music_1, "Music_1"),
  (music_2, "Music_2"),
  (music_3, "Music_3"),
  (jingles, "Jingles"),
])

And the result

{
  "source.fail_0": {
    "remainingTime": 0,
    "clockTime": 35.56,
    "isReady": false,
    "isFallible": true,
    "namespace": "Jingles"
  },
  "source.fail_3": {
    "remainingTime": 0,
    "clockTime": 35.56,
    "isReady": false,
    "isFallible": true,
    "namespace": "Music_3"
  },
  "source.fail_2": {
    "remainingTime": 0,
    "clockTime": 35.56,
    "isReady": false,
    "isFallible": true,
    "namespace": "Music_2"
  },
  "source.fail_1": {
    "remainingTime": 0,
    "clockTime": 35.56,
    "isReady": false,
    "isFallible": true,
    "namespace": "Music_1"
  },
  "single_0": {
    "remainingTime": 254.730997732,
    "clockTime": 35.56,
    "isReady": true,
    "isFallible": false,
    "namespace": "Security"
  },
  "switch_3": {
    "remainingTime": 0,
    "clockTime": 35.56,
    "isReady": false,
    "isFallible": true,
    "namespace": "Schedule"
  },
  "switch_6": {
    "remainingTime": null,
    "clockTime": 35.56,
    "isReady": true,
    "isFallible": false,
    "namespace": "Full"
  },
  "add_0": {
    "remainingTime": null,
    "clockTime": 35.56,
    "isReady": true,
    "isFallible": false,
    "namespace": "Radio"
  }
}

But if I try to uncomment

# let resultRecord.success = true

then I'll get

ERROR: (Failure "TODO: JSON of method not yet implemented")

Maybe I'm not so good at creating json interfaces, and result should look like

{
	"success": true,
	"sources": {
		"add_0": {
			"clockTime": 35.56,
			"isReady": true,
			"issFallible": false,
			"namespace": "Radio"
		}
	}
}

because I can then iterate over

if(obj.success === true) {
	Object.Entries(obj.sources).forEach(doSomething)
}

But It's possible to do that even if use object as is

if(obj.success === true) {
	Object.Entries(obj).filter((key) => key !== 'success').forEach(dosomething)
}

[toots]

No, this is a confusing situation and it's about to be clarified with #1824.

Rather than making sense of something tricky that's about to go away, I think it makes sense to explain how the script would be updated after that PR has been merged. Here we go:

def generateSourceInfoRecord(resultRecord, sourceNamePair)
	  let (src, name) = sourceNamePair
	  
	  recordItem = json()
	  
	  sourceId = source.id(src)
	  
	  recordItem.add("namespace", name)
	  recordItem.add("isFallible", source.fallible(src))
	  recordItem.add("isReady", source.is_ready(src))
	  recordItem.add("clockTime", source.time(src))
	  
	  remainingTime = source.remaining(src)
	  recordItem.add("remainingTime", (remainingTime != infinity) ? remainingTime : null())
	  
          resultRecord.add(sourceId, recordItem)
end

def generateResponse(sourceNamePairs)
           resultRecord = json()
           
	  resultRecord = list.iter(generateSourceInfoRecord(resultRecord), sourceNamePairs)
	  
	  resultRecord.add("success", true)
	  
	  json_of(compact = true, resultRecord)
end

generateResponse([
  (radio, "Radio"),
  (full, "Full"),
  (schedule, "Schedule"),
  (security, "Security"),
  (music_1, "Music_1"),
  (music_2, "Music_2"),
  (music_3, "Music_3"),
  (jingles, "Jingles"),
])

This should feel much more familiar, right?