diff --git a/common-pipeline/pipeline-flow/examples/pipeline-flow-v1-example.png b/common-pipeline/pipeline-flow/examples/pipeline-flow-supernode-diagram.png similarity index 100% rename from common-pipeline/pipeline-flow/examples/pipeline-flow-v1-example.png rename to common-pipeline/pipeline-flow/examples/pipeline-flow-supernode-diagram.png diff --git a/common-pipeline/pipeline-flow/examples/pipeline-flow-v1-example.snagproj b/common-pipeline/pipeline-flow/examples/pipeline-flow-supernode-diagram.snagproj similarity index 100% rename from common-pipeline/pipeline-flow/examples/pipeline-flow-v1-example.snagproj rename to common-pipeline/pipeline-flow/examples/pipeline-flow-supernode-diagram.snagproj diff --git a/common-pipeline/pipeline-flow/examples/pipeline-flow-v1-port-defs.png b/common-pipeline/pipeline-flow/examples/pipeline-flow-supernode-ports-diagram.png similarity index 100% rename from common-pipeline/pipeline-flow/examples/pipeline-flow-v1-port-defs.png rename to common-pipeline/pipeline-flow/examples/pipeline-flow-supernode-ports-diagram.png diff --git a/common-pipeline/pipeline-flow/examples/pipeline-flow-v1-port-defs.snagproj b/common-pipeline/pipeline-flow/examples/pipeline-flow-supernode-ports-diagram.snagproj similarity index 100% rename from common-pipeline/pipeline-flow/examples/pipeline-flow-v1-port-defs.snagproj rename to common-pipeline/pipeline-flow/examples/pipeline-flow-supernode-ports-diagram.snagproj diff --git a/common-pipeline/pipeline-flow/examples/pipeline-flow-v2-example-simple.json b/common-pipeline/pipeline-flow/examples/pipeline-flow-v2-example-simple.json deleted file mode 100644 index 6b65eb3..0000000 --- a/common-pipeline/pipeline-flow/examples/pipeline-flow-v2-example-simple.json +++ /dev/null @@ -1,214 +0,0 @@ -{ - "doc_type": "pipeline", - "version": "2.0", - "id": "24cc9881-f83a-4d38-b15b-8bf541e100fb", - "app_data": { - "ui_data": { - "name": "Simple Pipeline Flow Example", - "description": "A basic example pipeline - just does load/filter/save", - "class_name": "pipeline-flow-overall" - }, - "wml_data": { - "typeId": "stream" - } - }, - "primary_pipeline": "simple-pipeline", - "pipelines": [ - { - "id": "simple-pipeline", - "runtime_ref": "scala-spark-2.0.1", - "app_data": { - "ui_data": { - "zoom": 100, - "comments": [] - } - }, - "nodes": [ - { - "id": "entryID1PE", - "type": "binding", - "outputs": [ - { - "id": "entryPort1PE", - "schema_ref": "schema1" - } - ], - "connection": { - "ref": "d7c75790-90fc-4883-a5fc-3bb589166a52", - "project_ref": "0a0a20e8-e8a8-4710-b90a-6df3ff775b04", - "properties": { - "schema_name": "GOSALES", - "table_name": "PRODUCT_NAME_LOOKUP" - } - }, - "app_data": { - "ui_data": { - "x_pos": 10, - "y_pos": 10 - } - } - }, - { - "id": "nodeID2PE", - "type": "execution_node", - "op": "org.apache.spark.ml.ibm.transformers.FilterColumns", - "inputs": [ - { - "id": "input1NodeID2PE", - "schema_ref": "schema1", - "links": [ - { - "node_id_ref": "entryID1PE" - } - ], - "app_data": { - "ui_data": { - "label": "input 1 Node ID2PE" - } - } - } - ], - "outputs": [ - { - "id": "output1NodeID2PE", - "schema_ref": "schema2", - "app_data": { - "ui_data": { - "label": "output 1 Node ID2PE" - } - } - } - ], - "parameters": { - "dropColumns": [ - "integer2", - "double4" - ] - }, - "app_data": { - "ui_data": { - "label": "Filter", - "description": "Filter columns", - "image": "filter-columns.svg", - "x_pos": 189, - "y_pos": 56 - }, - "wml_data": { - "typeId": "filter-columns" - } - } - }, - { - "id": "exitID1PE", - "type": "binding", - "inputs": [ - { - "id": "exitPort1PE", - "schema_ref": "schema2", - "links": [ - { - "node_id_ref": "nodeID2PE" - } - ], - "app_data": { - "ui_data": { - "label": "exit Port 1PE" - } - } - } - ], - "data_asset": { - "properties": { - "name": "employee_data.csv" - } - }, - "app_data": { - "ui_data": { - "x_pos": 357, - "y_pos": 121 - } - } - } - ] - } - ], - "schemas": [ - { - "id": "schema1", - "fields": [ - { - "name": "integer2", - "type": "integer", - "nullable": false, - "metadata": { - "description": "Field integer2 is an integer field", - "measure": "range", - "modeling_role": "input", - "range": { - "min": -1, - "max": 400 - } - } - }, - { - "name": "double4", - "type": "double", - "nullable": false, - "metadata": { - "description": "Field double4 is a double field", - "measure": "range", - "modeling_role": "input", - "range": { - "min": 0.101, - "max": 100.211 - } - } - }, - { - "name": "string1", - "type": "string", - "nullable": true, - "metadata": { - "description": "Field string1 is a string field", - "measure": "set", - "modeling_role": "input", - "max_length": 24, - "values": [ - "One", - "Two", - "Three" - ] - } - } - ] - }, - { - "id": "schema2", - "fields": [ - { - "name": "string1", - "type": "string", - "nullable": true, - "metadata": { - "description": "Field string1 is a string field", - "measure": "set", - "modeling_role": "input", - "max_length": 24, - "values": [ - "One", - "Two", - "Three" - ] - } - } - ] - } - ], - "runtimes": [ - { - "id": "scala-spark-2.0.1", - "name": "Scala-Spark", - "version": "2.0.1" - } - ] -} diff --git a/common-pipeline/pipeline-flow/examples/pipeline-flow-v2-example.json b/common-pipeline/pipeline-flow/examples/pipeline-flow-v2-example.json deleted file mode 100644 index 00b9060..0000000 --- a/common-pipeline/pipeline-flow/examples/pipeline-flow-v2-example.json +++ /dev/null @@ -1,755 +0,0 @@ -{ - "doc_type": "pipeline", - "version": "2.0", - "json_schema": "https://api.dataplatform.ibm.com/schemas/common-pipeline/pipeline-flow/pipeline-flow-v2-schema.json", - "id": "24cc9881-f83a-4d38-b15b-8bf541e100fa", - "app_data": { - "ui_data": { - "name": "Pipeline Flow Example", - "description": "An example pipeline flow document with a sub-pipeline", - "class_name": "pipeline-flow-overall" - }, - "wml_data": { - "typeId": "stream" - } - }, - "primary_pipeline": "primary-pipeline", - "pipelines": [ - { - "id": "primary-pipeline", - "runtime_ref": "scala-spark-2.0.1", - "app_data": { - "ui_data": { - "zoom": 100, - "comments": [ - { - "id": "4e32ba0a-b234-3e33-b321-d693bf8f8452", - "x_pos": 232, - "y_pos": 223, - "width": 150, - "height": 50, - "content": "This is a comment about the comments array. The text should wrap within a bounded rectangle.", - "associated_id_refs": [ - { - "node_ref": "nodeIDSuperNodePE" - } - ] - } - ] - } - }, - "nodes": [ - { - "id": "entryID1PE", - "type": "binding", - "outputs": [ - { - "id": "entryPort1PE", - "schema_ref": "schema1", - "app_data": { - "ui_data": { - "label": "exit Port 1PE" - } - } - } - ], - "app_data": { - "ui_data": { - "x_pos": 10, - "y_pos": 10, - "ui_parameters": { - "uiOnlyField1": "test", - "uiOnlyField2": [ - "alpha", - "beta", - "gamma", - "delta" - ], - "uiOnlyField3": 23 - } - } - } - }, - { - "id": "entryID2PE", - "type": "binding", - "outputs": [ - { - "id": "entryPort2PE", - "schema_ref": "schema2", - "app_data": { - "ui_data": { - "label": "exit Port 1PE" - } - } - } - ], - "app_data": { - "ui_data": { - "x_pos": 8.002765655517578, - "y_pos": 119.84760284423828 - } - } - }, - { - "id": "entryID3PE", - "type": "binding", - "outputs": [ - { - "id": "entryPort3PE", - "schema_ref": "schema2", - "app_data": { - "ui_data": { - "label": "entryID3PE" - } - } - } - ], - "connection": { - "ref": "d7c75790-90fc-4883-a5fc-3bb589166a52", - "project_ref": "0a0a20e8-e8a8-4710-b90a-6df3ff775b04", - "properties": { - "schema_name": "GOSALES", - "table_name": "PRODUCT_NAME_LOOKUP" - } - }, - "app_data": { - "ui_data": { - "x_pos": 9.001384735107422, - "y_pos": 214.71601104736328 - } - } - }, - { - "id": "nodeID1PE", - "type": "execution_node", - "op": "org.apache.spark.ml.ibm.transformers.Join", - "inputs": [ - { - "id": "input1NodeID1PE", - "links": [ - { - "node_id_ref": "entryID1PE", - "app_data": { - "ui_data": { - "class_name": "d3-data-link" - } - } - } - ], - "schema_ref": "schema1", - "app_data": { - "ui_data": { - "cardinality": { - "min": 1, - "max": 1 - }, - "label": "input1NodeID1PE" - } - } - }, - { - "id": "input2NodeID1PE", - "links": [ - { - "node_id_ref": "entryID2PE", - "app_data": { - "ui_data": { - "class_name": "d3-data-link" - } - } - } - ], - "schema_ref": "schema2", - "app_data": { - "ui_data": { - "cardinality": { - "min": 1, - "max": 1 - }, - "label": "input2NodeID1PE" - } - } - } - ], - "outputs": [ - { - "id": "output1NodeID1PE", - "schema_ref": "schema3", - "app_data": { - "ui_data": { - "cardinality": { - "min": 1, - "max": -1 - }, - "label": "output1NodeID1PE" - } - } - } - ], - "parameters": { - "join-type": "inner", - "key1": "inputDS1.NAME", - "key2": "inputDS2.name" - }, - "app_data": { - "ui_data": { - "label": "Join", - "description": "Join two or more data records", - "image": "join.svg", - "x_pos": 167.89193725585938, - "y_pos": 32.24104166030884, - "attributes": "attr1='value1'; attr2='value2'", - "decorations": [ - { - "position": "topRight", - "class_name": "zoom-in-decoration", - "hotspot": true, - "id": "zoomInDecorator", - "image": "/images/decorators/zoom-in_32.svg" - } - ] - }, - "wml_data": { - "typeId": "Join" - } - } - }, - { - "id": "nodeIDSuperNodePE", - "type": "super_node", - "subflow_ref": { - "pipeline_id_ref": "modeler-sub-pipeline" - }, - "inputs": [ - { - "id": "input1SuperNodePE", - "links": [ - { - "node_id_ref": "nodeID1PE", - "app_data": { - "ui_data": { - "class_name": "d3-data-link" - } - } - } - ], - "subflow_node_ref": "entryID1SE", - "schema_ref": "schema3", - "app_data": { - "ui_data": { - "class_name": "port-class", - "label": "input1SuperNodePE" - } - } - }, - { - "id": "input2SuperNodePE", - "links": [ - { - "node_id_ref": "entryID3PE", - "app_data": { - "ui_data": { - "class_name": "d3-data-link" - } - } - } - ], - "subflow_node_ref": "entryID2SE", - "schema_ref": "schema2", - "app_data": { - "ui_data": { - "class_name": "port-class", - "label": "input2SuperNodePE" - } - } - } - ], - "outputs": [ - { - "id": "output1SuperNodePE", - "schema_ref": "schema2", - "subflow_node_ref": "exitID1SE", - "app_data": { - "ui_data": { - "class_name": "port-class", - "label": "output1SuperNodePE" - } - } - } - ], - "app_data": { - "ui_data": { - "x_pos": 317.99029541015625, - "y_pos": 46.97645568847656 - } - } - }, - { - "id": "nodeID2PE", - "type": "execution_node", - "op": "org.apache.spark.ml.ibm.transformers.Filter", - "inputs": [ - { - "id": "input1NodeID2PE", - "links": [ - { - "node_id_ref": "nodeIDSuperNodePE", - "port_id_ref": "output1SuperNodePE" - } - ], - "schema_ref": "schema2", - "app_data": { - "ui_data": { - "cardinality": { - "min": 1, - "max": 1 - }, - "class_name": "port-class", - "label": "input1NodeID2PE" - } - } - } - ], - "outputs": [ - { - "id": "output1NodeID2PE", - "schema_ref": "schema3", - "app_data": { - "ui_data": { - "class_name": "port-class", - "label": "output1NodeID2PE" - } - } - } - ], - "parameters": { - "conditionExpr": "Age > 20" - }, - "app_data": { - "ui_data": { - "label": "Filter", - "description": "Filter record data", - "image": "filter-rows.svg", - "x_pos": 457.16619873046875, - "y_pos": 199.00416564941406 - }, - "wml_data": { - "typeId": "filter-records" - } - } - }, - { - "id": "nodeID3PE", - "type": "execution_node", - "op": "org.apache.spark.ml.ibm.transformers.Filter", - "inputs": [ - { - "id": "input1NodeID3PE", - "links": [ - { - "node_id_ref": "nodeIDSuperNodePE" - } - ], - "schema_ref": "schema2", - "app_data": { - "ui_data": { - "cardinality": { - "min": 1, - "max": -1 - }, - "class_name": "port-class", - "label": "input1NodeID3PE" - } - } - } - ], - "outputs": [ - { - "id": "output1NodeID3PE", - "schema_ref": "schema3", - "app_data": { - "ui_data": { - "class_name": "port-class", - "label": "output1NodeID3PE" - } - } - } - ], - "parameters": { - "conditionExpr": "Age > ${age}" - }, - "app_data": { - "ui_data": { - "label": "Filter", - "description": "Filter record data", - "image": "filter-rows.svg", - "x_pos": 548.6466674804688, - "y_pos": 63.175933837890625 - }, - "wml_data": { - "typeId": "filter-records" - } - } - }, - { - "id": "exitID1PE", - "type": "binding", - "inputs": [ - { - "id": "exitPort1PE", - "schema_ref": "schema3", - "links": [ - { - "node_id_ref": "nodeID2PE", - "app_data": { - "ui_data": { - "class_name": "d3-data-link" - } - } - } - ], - "app_data": { - "ui_data": { - "label": "exitPort1PE" - } - } - } - ], - "app_data": { - "ui_data": { - "x_pos": 760.0665283203125, - "y_pos": 203.84069061279297 - } - } - }, - { - "id": "exitID2PE", - "type": "binding", - "inputs": [ - { - "id": "exitPort2PE", - "schema_ref": "schema3", - "links": [ - { - "node_id_ref": "nodeID3PE", - "app_data": { - "ui_data": { - "class_name": "d3-data-link" - } - } - } - ], - "app_data": { - "ui_data": { - "label": "exitID2PE" - } - } - } - ], - "app_data": { - "ui_data": { - "x_pos": 774.8226928710938, - "y_pos": 100.94320678710938 - } - } - } - ] - }, - { - "id": "modeler-sub-pipeline", - "runtime_ref": "SPSS-Modeler", - "app_data": { - "ui_data": { - "zoom": 150, - "comments": [] - } - }, - "nodes": [ - { - "id": "entryID1SE", - "type": "binding", - "outputs": [ - { - "id": "entryPort1SE", - "schema_ref": "schema3", - "app_data": { - "ui_data": { - "label": "entryPort1SE" - } - } - } - ] - }, - { - "id": "entryID2SE", - "type": "binding", - "outputs": [ - { - "id": "entryPort2SE", - "schema_ref": "schema2", - "app_data": { - "ui_data": { - "label": "entryPort2SE" - } - } - } - ] - }, - { - "id": "nodeID1SE", - "type": "execution_node", - "op": "org.apache.spark.ml.ibm.transformers.Filter", - "inputs": [ - { - "id": "input1nodeID1SE", - "links": [ - { - "node_id_ref": "entryID1SE" - } - ], - "schema_ref": "schema3", - "app_data": { - "ui_data": { - "class_name": "port-class", - "label": "input1nodeID1SE" - } - } - } - ], - "outputs": [ - { - "id": "output1nodeID1SE", - "schema_ref": "schema3", - "app_data": { - "ui_data": { - "class_name": "port-class", - "label": "output1nodeID1SE" - } - } - } - ], - "parameters": { - "conditionExpr": "Age > ${age}" - }, - "app_data": { - "ui_data": { - "label": "Filter", - "description": "Filter record data", - "image": "filter-rows.svg", - "x_pos": 784, - "y_pos": 202 - }, - "wml_data": { - "typeId": "filter-records" - } - } - }, - { - "id": "nodeID2SE", - "type": "execution_node", - "op": "org.apache.spark.ml.ibm.transformers.Join", - "inputs": [ - { - "id": "input1NodeID2SE", - "links": [ - { - "node_id_ref": "nodeID1SE" - } - ], - "schema_ref": "schema3", - "app_data": { - "ui_data": { - "cardinality": { - "min": 1, - "max": 1 - }, - "label": "input1NodeID2SE" - } - } - }, - { - "id": "input2NodeID2SE", - "links": [ - { - "node_id_ref": "entryID2SE" - } - ], - "schema_ref": "schema2", - "app_data": { - "ui_data": { - "cardinality": { - "min": 1, - "max": 1 - }, - "label": "input2NodeID2SE" - } - } - } - ], - "outputs": [ - { - "id": "output1NodeID2SE", - "schema_ref": "schema2", - "app_data": { - "ui_data": { - "label": "output1NodeID2SE" - } - } - } - ], - "parameters": { - "join-type": "inner", - "key1": "inputDS1.NAME", - "key2": "inputDS2.name" - }, - "app_data": { - "ui_data": { - "label": "Join", - "description": "Join two or more data records", - "image": "join.svg", - "x_pos": 651, - "y_pos": 70, - "attributes": "attr1='value1'; attr2='value2'" - }, - "wml_data": { - "typeId": "Join" - } - } - }, - { - "id": "exitID1SE", - "type": "binding", - "inputs": [ - { - "id": "exitPort1SE", - "schema_ref": "schema2", - "links": [ - { - "node_id_ref": "nodeID2SE" - } - ] - } - ] - } - ] - } - ], - "schemas": [ - { - "id": "schema1", - "fields": [ - { - "name": "integer2", - "type": "integer", - "nullable": false, - "metadata": { - "description": "Field integer2 is an integer field", - "measure": "range", - "modeling_role": "input", - "range": { - "min": -1, - "max": 400 - } - } - }, - { - "name": "double4", - "type": "double", - "nullable": false, - "metadata": { - "description": "Field double4 is a double field", - "measure": "range", - "modeling_role": "input", - "range": { - "min": 0.101, - "max": 100.211 - } - } - } - ] - }, - { - "id": "schema2", - "fields": [ - { - "name": "string1", - "type": "string", - "nullable": true, - "metadata": { - "description": "Field string1 is a string field", - "measure": "set", - "modeling_role": "input", - "max_length": 24, - "values": [ - "One", - "Two", - "Three" - ] - } - } - ] - }, - { - "id": "schema3", - "fields": [ - { - "name": "integer2", - "type": "integer", - "nullable": false, - "metadata": { - "description": "Field integer2 is an integer field", - "measure": "range", - "modeling_role": "input", - "range": { - "min": -1, - "max": 400 - } - } - }, - { - "name": "double4", - "type": "double", - "nullable": false, - "metadata": { - "description": "Field double4 is a double field", - "measure": "range", - "modeling_role": "input", - "range": { - "min": 0.101, - "max": 100.211 - } - } - }, - { - "name": "string1", - "type": "string", - "nullable": true, - "metadata": { - "description": "Field string1 is a string field", - "measure": "set", - "modeling_role": "input", - "max_length": 24, - "values": [ - "One", - "Two", - "Three" - ] - } - } - ] - } - ], - "runtimes": [ - { - "id": "scala-spark-2.0.1", - "name": "Scala-Spark", - "version": "2.0.1" - }, - { - "id": "SPSS-Modeler", - "name": "SPSS Modeler", - "version": "18.0" - } - ] -} diff --git a/common-pipeline/pipeline-flow/examples/pipeline-flow-v2-external-subflow-example.json b/common-pipeline/pipeline-flow/examples/pipeline-flow-v2-external-subflow-example.json deleted file mode 100644 index 8ab6bc6..0000000 --- a/common-pipeline/pipeline-flow/examples/pipeline-flow-v2-external-subflow-example.json +++ /dev/null @@ -1,191 +0,0 @@ -{ - "doc_type": "pipeline", - "version": "2.0", - "id": "abdc9881-f83a-4d38-b15b-8bf541e100fb", - "app_data": { - "ui_data": { - "name": "External Sub-Flow Example", - "description": "An example using a supernode that refers to an external sub-pipeline", - "class_name": "pipeline-flow-overall" - }, - "wml_data": { - "typeId": "stream" - } - }, - "primary_pipeline": "external-sub-flow-pipeline", - "pipelines": [ - { - "id": "external-sub-flow-pipeline", - "runtime_ref": "scala-spark-2.0.1", - "app_data": { - "ui_data": { - "zoom": 100, - "comments": [] - } - }, - "nodes": [ - { - "id": "entryID1", - "type": "binding", - "outputs": [ - { - "id": "entryPort1PE", - "schema_ref": "schema1", - "app_data": { - "ui_data": { - "label": "entryPort1PE" - } - } - } - ] - }, - { - "id": "superNode1", - "type": "super_node", - "sub_type": "shaper", - "subflow_ref": { - "url": "https://github.com/elyra-ai/pipeline-schemas/blob/main/common-pipeline/pipeline-flow/examples/pipeline-flow-v2-example-simple.json", - "pipeline_id_ref": "simple-pipeline" - }, - "inputs": [ - { - "id": "input1SuperNodePE", - "subflow_node_ref": "entryID1PE", - "links": [ - { - "node_id_ref": "entryID1" - } - ], - "schema_ref": "schema1", - "app_data": { - "ui_data": { - "class_name": "port-class", - "label": "Input Port" - } - } - } - ], - "outputs": [ - { - "id": "output1SuperNodePE", - "subflow_node_ref": "exitID1PE", - "schema_ref": "schema2", - "app_data": { - "ui_data": { - "class_name": "port-class", - "label": "Output port" - } - } - } - ], - "app_data": { - "ui_data": { - "x_pos": 229, - "y_pos": 88 - } - } - }, - { - "id": "exitID1", - "type": "binding", - "inputs": [ - { - "id": "exitPort1", - "schema_ref": "schema2", - "links": [ - { - "node_id_ref": "superNode1" - } - ] - } - ], - "app_data": { - "ui_data": { - "x_pos": 446, - "y_pos": 69 - } - } - } - ] - } - ], - "schemas": [ - { - "id": "schema1", - "fields": [ - { - "name": "integer2", - "type": "integer", - "nullable": false, - "metadata": { - "description": "Field integer2 is an integer field", - "measure": "range", - "modeling_role": "input", - "range": { - "min": -1, - "max": 400 - } - } - }, - { - "name": "double4", - "type": "double", - "nullable": false, - "metadata": { - "description": "Field double4 is a double field", - "measure": "range", - "modeling_role": "input", - "range": { - "min": 0.101, - "max": 100.211 - } - } - }, - { - "name": "string1", - "type": "string", - "nullable": true, - "metadata": { - "description": "Field string1 is a string field", - "measure": "set", - "modeling_role": "input", - "max_length": 24, - "values": [ - "One", - "Two", - "Three" - ] - } - } - ] - }, - { - "id": "schema2", - "fields": [ - { - "name": "string1", - "type": "string", - "nullable": true, - "metadata": { - "description": "Field string1 is a string field", - "measure": "set", - "modeling_role": "input", - "max_length": 24, - "values": [ - "One", - "Two", - "Three" - ] - } - } - ] - } - ], - "runtimes": [ - { - "id": "scala-spark-2.0.1", - "name": "Scala-Spark", - "version": "2.0.1" - } - ] -} diff --git a/common-pipeline/pipeline-flow/examples/pipeline-flow-v2-modeling-example.json b/common-pipeline/pipeline-flow/examples/pipeline-flow-v2-modeling-example.json deleted file mode 100644 index 3318dbd..0000000 --- a/common-pipeline/pipeline-flow/examples/pipeline-flow-v2-modeling-example.json +++ /dev/null @@ -1,347 +0,0 @@ -{ - "doc_type": "pipeline", - "version": "2.0", - "id": "e456f213-f83a-4d38-b15b-8bf541e100fb", - "app_data": { - "ui_data": { - "name": "Modeling Pipeline Flow Example", - "description": "An example ML pipeline - provides load/filter/model-build with an already-built model", - "class_name": "pipeline-flow-overall" - }, - "wml_data": { - "typeId": "stream" - } - }, - "primary_pipeline": "modeling-pipeline", - "pipelines": [ - { - "id": "modeling-pipeline", - "runtime_ref": "scala-spark-2.0.1", - "app_data": { - "ui_data": { - "zoom": 100, - "comments": [] - } - }, - "nodes": [ - { - "id": "entryID1PE", - "type": "binding", - "outputs": [ - { - "id": "entryPort1PE", - "schema_ref": "schema1", - "app_data": { - "ui_data": { - "label": "Output Port" - } - } - } - ], - "app_data": { - "ui_data": { - "label": "entryID1PE" - } - } - }, - { - "id": "filter_nodeID2PE", - "type": "execution_node", - "op": "org.apache.spark.ml.ibm.transformers.Filter", - "inputs": [ - { - "id": "input1NodeID2PE", - "links": [ - { - "node_id_ref": "nodeIDSuperNodePE", - "port_id_ref": "output1SuperNodePE" - } - ], - "schema_ref": "schema2", - "app_data": { - "ui_data": { - "cardinality": { - "min": 1, - "max": 1 - }, - "class_name": "port-class", - "label": "Input Port" - } - } - } - ], - "outputs": [ - { - "id": "output1NodeID2PE", - "schema_ref": "schema1", - "app_data": { - "ui_data": { - "class_name": "port-class", - "label": "Output port" - } - } - } - ], - "parameters": { - "conditionExpr": "integer2 > 20" - }, - "app_data": { - "ui_data": { - "label": "Filter", - "description": "Filter record data", - "image": "filter-rows.svg", - "x_pos": 66, - "y_pos": 144 - }, - "wml_data": { - "typeId": "filter-records" - } - } - }, - { - "id": "modeling_nodeID3PE", - "type": "execution_node", - "op": "org.apache.spark.ml.regression.LinearRegression", - "inputs": [ - { - "id": "input1modeling_nodeID3PE", - "schema_ref": "schema1", - "links": [ - { - "node_id_ref": "filter_nodeID2PE" - } - ], - "app_data": { - "ui_data": { - "class_name": "portClass", - "label": "input1modeling_nodeID3PE" - } - } - } - ], - "outputs": [ - { - "id": "output1modeling_nodeID3PE", - "schema_ref": "schema1", - "app_data": { - "ui_data": { - "class_name": "portClass", - "label": "output1modeling_nodeID3PE" - } - } - } - ], - "parameters": { - "targetField": [ - "integer2" - ], - "inputFieldList": [ - "double4", - "string1" - ], - "weightCol": [], - "elasticNetParam": 0, - "fitIntercept": false, - "maxIter": 90, - "regParam": 0.1, - "solver": "auto", - "standardization": true, - "tol": 0.000001 - }, - "app_data": { - "ui_data": { - "label": "Linear Regression", - "description": "Fit a Linear Regression Model", - "image": "linearregression.svg", - "x_pos": 405, - "y_pos": 187, - "associations": [ - { - "id": "link_model_nodeID4PE", - "node_ref": "model_nodeID4PE" - } - ] - }, - "wml_data": { - "typeId": "linearregression", - "associations": [ - { - "ui_ref": "link_model_nodeID4PE", - "node_ref": "model_nodeID4PE", - "type": "trains_model" - } - ] - } - } - }, - { - "id": "model_nodeID4PE", - "type": "model_node", - "model_ref": "https://reference.to.model.blob.in.storage", - "inputs": [ - { - "id": "input1NodeID4PE", - "schema_ref": "schema1", - "links": [ - { - "node_id_ref": "entryID1PE" - } - ], - "app_data": { - "ui_data": { - "label": "input1NodeID4PE" - } - } - } - ], - "outputs": [ - { - "id": "output1NodeID4PE", - "schema_ref": "schema2", - "app_data": { - "ui_data": { - "class_name": "port-class", - "label": "Output port" - } - } - } - ], - "parameters": {}, - "app_data": { - "ui_data": { - "label": "Linear Regression Model", - "description": "Fitted Linear Regression Model", - "image": "modelnuggetspark.svg", - "x_pos": 273, - "y_pos": 35 - }, - "wml_data": { - "typeId": "linearregressionmodel" - } - } - } - ] - } - ], - "schemas": [ - { - "id": "schema1", - "fields": [ - { - "name": "integer2", - "type": "integer", - "nullable": false, - "metadata": { - "description": "Field integer2 is an integer field", - "measure": "range", - "modeling_role": "input", - "range": { - "min": -1, - "max": 400 - } - } - }, - { - "name": "double4", - "type": "double", - "nullable": false, - "metadata": { - "description": "Field double4 is a double field", - "measure": "range", - "modeling_role": "input", - "range": { - "min": 0.101, - "max": 100.211 - } - } - }, - { - "name": "string1", - "type": "string", - "nullable": true, - "metadata": { - "description": "Field string1 is a string field", - "measure": "set", - "modeling_role": "input", - "max_length": 24, - "values": [ - "One", - "Two", - "Three" - ] - } - } - ] - }, - { - "id": "schema2", - "fields": [ - { - "name": "integer2", - "type": "integer", - "nullable": false, - "metadata": { - "description": "Field integer2 is an integer field", - "measure": "range", - "modeling_role": "input", - "range": { - "min": -1, - "max": 400 - } - } - }, - { - "name": "double4", - "type": "double", - "nullable": false, - "metadata": { - "description": "Field double4 is a double field", - "measure": "range", - "modeling_role": "input", - "range": { - "min": 0.101, - "max": 100.211 - } - } - }, - { - "name": "string1", - "type": "string", - "nullable": true, - "metadata": { - "description": "Field string1 is a string field", - "measure": "set", - "modeling_role": "input", - "max_length": 24, - "values": [ - "One", - "Two", - "Three" - ] - } - }, - { - "name": "$integer2_predicted", - "type": "integer", - "nullable": false, - "metadata": { - "description": "Predicted field for 'integer2'", - "measure": "range", - "modeling_role": "input", - "range": { - "min": -1, - "max": 400 - } - } - } - ] - } - ], - "runtimes": [ - { - "id": "scala-spark-2.0.1", - "name": "Scala-Spark", - "version": "2.0.1" - } - ] -} diff --git a/common-pipeline/pipeline-flow/examples/pipeline-flow-v3-example-simple.json b/common-pipeline/pipeline-flow/examples/pipeline-flow-v3-example-simple.json new file mode 100644 index 0000000..ef035d5 --- /dev/null +++ b/common-pipeline/pipeline-flow/examples/pipeline-flow-v3-example-simple.json @@ -0,0 +1,232 @@ +{ + "doc_type": "pipeline", + "version": "3.0", + "id": "24cc9881-f83a-4d38-b15b-8bf541e100fb", + "app_data": { + "ui_data": { + "name": "Simple Pipeline Flow Example", + "description": "A basic example pipeline - just does load/filter/save", + "class_name": "pipeline-flow-overall" + }, + "wml_data": { + "typeId": "stream" + } + }, + "primary_pipeline": "simple-pipeline", + "pipelines": [ + { + "id": "simple-pipeline", + "nodes": [ + { + "id": "entryID1PE", + "type": "binding", + "op": "bind_input", + "app_data": { + "ui_data": { + "label": "Load", + "image": "", + "x_pos": 10, + "y_pos": 10 + } + }, + "outputs": [ + { + "id": "entryPort1PE", + "schema_ref": "schema1", + "app_data": { + "ui_data": { + "label": "" + } + } + } + ], + "connection": { + "ref": "d7c75790-90fc-4883-a5fc-3bb589166a52", + "project_ref": "0a0a20e8-e8a8-4710-b90a-6df3ff775b04", + "properties": { + "schema_name": "GOSALES", + "table_name": "PRODUCT_NAME_LOOKUP" + } + } + }, + { + "id": "nodeID2PE", + "type": "execution_node", + "op": "org.apache.spark.ml.ibm.transformers.FilterColumns", + "app_data": { + "wml_data": { + "typeId": "filter-columns" + }, + "ui_data": { + "label": "Filter", + "image": "", + "x_pos": 189, + "y_pos": 56, + "description": "Filter columns" + } + }, + "inputs": [ + { + "id": "input1NodeID2PE", + "schema_ref": "schema1", + "app_data": { + "ui_data": { + "label": "input 1 Node ID2PE" + } + }, + "links": [ + { + "id": "3f91ebcb-1357-40e5-bab3-55a93a444601", + "node_id_ref": "entryID1PE" + } + ] + } + ], + "outputs": [ + { + "id": "output1NodeID2PE", + "schema_ref": "schema2", + "app_data": { + "ui_data": { + "label": "output 1 Node ID2PE" + } + } + } + ], + "parameters": { + "dropColumns": [ + "integer2", + "double4" + ] + } + }, + { + "id": "exitID1PE", + "type": "binding", + "op": "bind_output", + "app_data": { + "ui_data": { + "label": "Save", + "image": "", + "x_pos": 357, + "y_pos": 121 + } + }, + "inputs": [ + { + "id": "exitPort1PE", + "schema_ref": "schema2", + "app_data": { + "ui_data": { + "label": "exit Port 1PE" + } + }, + "links": [ + { + "id": "01e100e6-6bba-4f97-9439-e935697e15bf", + "node_id_ref": "nodeID2PE" + } + ] + } + ], + "data_asset": { + "properties": { + "name": "employee_data.csv" + } + } + } + ], + "app_data": { + "ui_data": { + "zoom": { + "x": 0, + "y": 0, + "k": 1 + }, + "comments": [] + } + }, + "runtime_ref": "scala-spark-2.0.1" + } + ], + "schemas": [ + { + "id": "schema1", + "fields": [ + { + "name": "integer2", + "type": "integer", + "nullable": false, + "metadata": { + "description": "Field integer2 is an integer field", + "measure": "range", + "modeling_role": "input", + "range": { + "min": -1, + "max": 400 + } + } + }, + { + "name": "double4", + "type": "double", + "nullable": false, + "metadata": { + "description": "Field double4 is a double field", + "measure": "range", + "modeling_role": "input", + "range": { + "min": 0.101, + "max": 100.211 + } + } + }, + { + "name": "string1", + "type": "string", + "nullable": true, + "metadata": { + "description": "Field string1 is a string field", + "measure": "set", + "modeling_role": "input", + "max_length": 24, + "values": [ + "One", + "Two", + "Three" + ] + } + } + ] + }, + { + "id": "schema2", + "fields": [ + { + "name": "string1", + "type": "string", + "nullable": true, + "metadata": { + "description": "Field string1 is a string field", + "measure": "set", + "modeling_role": "input", + "max_length": 24, + "values": [ + "One", + "Two", + "Three" + ] + } + } + ] + } + ], + "runtimes": [ + { + "id": "scala-spark-2.0.1", + "name": "Scala-Spark", + "version": "2.0.1" + } + ], + "json_schema": "https://api.dataplatform.ibm.com/schemas/common-pipeline/pipeline-flow/pipeline-flow-v3-schema.json" + } diff --git a/common-pipeline/pipeline-flow/examples/pipeline-flow-v3-example.json b/common-pipeline/pipeline-flow/examples/pipeline-flow-v3-example.json new file mode 100644 index 0000000..9d6ae36 --- /dev/null +++ b/common-pipeline/pipeline-flow/examples/pipeline-flow-v3-example.json @@ -0,0 +1,809 @@ +{ + "doc_type": "pipeline", + "version": "3.0", + "json_schema": "https://api.dataplatform.ibm.com/schemas/common-pipeline/pipeline-flow/pipeline-flow-v3-schema.json", + "id": "24cc9881-f83a-4d38-b15b-8bf541e100fa", + "app_data": { + "ui_data": { + "name": "Pipeline Flow Example", + "description": "An example pipeline flow document with a sub-pipeline", + "class_name": "pipeline-flow-overall" + }, + "wml_data": { + "typeId": "stream" + } + }, + "primary_pipeline": "primary-pipeline", + "pipelines": [ + { + "id": "primary-pipeline", + "nodes": [ + { + "id": "entryID1PE", + "type": "binding", + "app_data": { + "ui_data": { + "label": "Load 1", + "x_pos": 10, + "y_pos": 10, + "ui_parameters": { + "uiOnlyField1": "test", + "uiOnlyField2": [ + "alpha", + "beta", + "gamma", + "delta" + ], + "uiOnlyField3": 23 + } + } + }, + "outputs": [ + { + "id": "entryPort1PE", + "schema_ref": "schema1", + "app_data": { + "ui_data": { + "label": "exit Port 1PE" + } + } + } + ] + }, + { + "id": "entryID2PE", + "type": "binding", + "app_data": { + "ui_data": { + "label": "Load 2", + "x_pos": 8.002765655517578, + "y_pos": 244.84760284423828 + } + }, + "outputs": [ + { + "id": "entryPort2PE", + "schema_ref": "schema2", + "app_data": { + "ui_data": { + "label": "exit Port 1PE" + } + } + } + ] + }, + { + "id": "entryID3PE", + "type": "binding", + "app_data": { + "ui_data": { + "label": "Load 3", + "x_pos": 9.001384735107422, + "y_pos": 339.7160110473633 + } + }, + "outputs": [ + { + "id": "entryPort3PE", + "schema_ref": "schema2", + "app_data": { + "ui_data": { + "label": "entryID3PE" + } + } + } + ], + "connection": { + "ref": "d7c75790-90fc-4883-a5fc-3bb589166a52", + "project_ref": "0a0a20e8-e8a8-4710-b90a-6df3ff775b04", + "properties": { + "schema_name": "GOSALES", + "table_name": "PRODUCT_NAME_LOOKUP" + } + } + }, + { + "id": "nodeID1PE", + "type": "execution_node", + "op": "org.apache.spark.ml.ibm.transformers.Join", + "app_data": { + "wml_data": { + "typeId": "Join" + }, + "ui_data": { + "label": "Join", + "image": "join.svg", + "x_pos": 167.89193725585938, + "y_pos": 32.24104166030884, + "description": "Join two or more data records", + "decorations": [ + { + "position": "topRight", + "class_name": "zoom-in-decoration", + "hotspot": true, + "id": "zoomInDecorator", + "image": "/images/decorators/zoom-in_32.svg" + } + ] + } + }, + "inputs": [ + { + "id": "input1NodeID1PE", + "schema_ref": "schema1", + "app_data": { + "ui_data": { + "cardinality": { + "min": 1, + "max": 1 + }, + "label": "input1NodeID1PE" + } + }, + "links": [ + { + "id": "e0e62a14-6107-4dba-83b8-827575d17335", + "node_id_ref": "entryID1PE", + "app_data": { + "ui_data": { + "class_name": "d3-data-link" + } + } + } + ] + }, + { + "id": "input2NodeID1PE", + "schema_ref": "schema2", + "app_data": { + "ui_data": { + "cardinality": { + "min": 1, + "max": 1 + }, + "label": "input2NodeID1PE" + } + }, + "links": [ + { + "id": "54fd09ff-17c9-4807-bdfe-143329f61b86", + "node_id_ref": "entryID2PE", + "app_data": { + "ui_data": { + "class_name": "d3-data-link" + } + } + } + ] + } + ], + "outputs": [ + { + "id": "output1NodeID1PE", + "schema_ref": "schema3", + "app_data": { + "ui_data": { + "cardinality": { + "min": 1, + "max": -1 + }, + "label": "output1NodeID1PE" + } + } + } + ], + "parameters": { + "join-type": "inner", + "key1": "inputDS1.NAME", + "key2": "inputDS2.name" + } + }, + { + "id": "nodeIDSuperNodePE", + "type": "super_node", + "subflow_ref": { + "pipeline_id_ref": "modeler-sub-pipeline" + }, + "app_data": { + "ui_data": { + "label": "Supernode", + "x_pos": 317.99029541015625, + "y_pos": 46.97645568847656, + "is_expanded": true, + "expanded_width": 200, + "expanded_height": 200 + } + }, + "inputs": [ + { + "id": "input1SuperNodePE", + "subflow_node_ref": "entryID1SE", + "schema_ref": "schema3", + "app_data": { + "ui_data": { + "class_name": "port-class", + "label": "input1SuperNodePE" + } + }, + "links": [ + { + "id": "edbb27d1-d94a-4532-9a6b-6f78bbebdb13", + "node_id_ref": "nodeID1PE", + "app_data": { + "ui_data": { + "class_name": "d3-data-link" + } + } + } + ] + }, + { + "id": "input2SuperNodePE", + "subflow_node_ref": "entryID2SE", + "schema_ref": "schema2", + "app_data": { + "ui_data": { + "class_name": "port-class", + "label": "input2SuperNodePE" + } + }, + "links": [ + { + "id": "14f08fac-e041-48ff-89d3-85cac13ecacc", + "node_id_ref": "entryID3PE", + "app_data": { + "ui_data": { + "class_name": "d3-data-link" + } + } + } + ] + } + ], + "outputs": [ + { + "id": "output1SuperNodePE", + "subflow_node_ref": "exitID1SE", + "schema_ref": "schema2", + "app_data": { + "ui_data": { + "class_name": "port-class", + "label": "output1SuperNodePE" + } + } + } + ] + }, + { + "id": "nodeID2PE", + "type": "execution_node", + "op": "org.apache.spark.ml.ibm.transformers.Filter", + "app_data": { + "wml_data": { + "typeId": "filter-records" + }, + "ui_data": { + "label": "Filter 1", + "image": "filter-rows.svg", + "x_pos": 587.1661987304688, + "y_pos": 324.00416564941406, + "description": "Filter record data" + } + }, + "inputs": [ + { + "id": "input1NodeID2PE", + "schema_ref": "schema2", + "app_data": { + "ui_data": { + "cardinality": { + "min": 1, + "max": 1 + }, + "class_name": "port-class", + "label": "input1NodeID2PE" + } + }, + "links": [ + { + "id": "7bed5dc3-617f-4683-99e4-d9759c754c79", + "node_id_ref": "nodeIDSuperNodePE", + "port_id_ref": "output1SuperNodePE" + } + ] + } + ], + "outputs": [ + { + "id": "output1NodeID2PE", + "schema_ref": "schema3", + "app_data": { + "ui_data": { + "class_name": "port-class", + "label": "output1NodeID2PE" + } + } + } + ], + "parameters": { + "conditionExpr": "Age > 20" + } + }, + { + "id": "nodeID3PE", + "type": "execution_node", + "op": "org.apache.spark.ml.ibm.transformers.Filter", + "app_data": { + "wml_data": { + "typeId": "filter-records" + }, + "ui_data": { + "label": "Filter 2", + "image": "filter-rows.svg", + "x_pos": 678.6466674804688, + "y_pos": 63.175933837890625, + "description": "Filter record data" + } + }, + "inputs": [ + { + "id": "input1NodeID3PE", + "schema_ref": "schema2", + "app_data": { + "ui_data": { + "cardinality": { + "min": 1, + "max": -1 + }, + "class_name": "port-class", + "label": "input1NodeID3PE" + } + }, + "links": [ + { + "id": "5c4f1667-6625-4f96-82ce-5b93af2b40b2", + "node_id_ref": "nodeIDSuperNodePE" + } + ] + } + ], + "outputs": [ + { + "id": "output1NodeID3PE", + "schema_ref": "schema3", + "app_data": { + "ui_data": { + "class_name": "port-class", + "label": "output1NodeID3PE" + } + } + } + ], + "parameters": { + "conditionExpr": "Age > ${age}" + } + }, + { + "id": "exitID1PE", + "type": "binding", + "app_data": { + "ui_data": { + "label": "Save 1", + "x_pos": 890.0665283203125, + "y_pos": 328.84069061279297 + } + }, + "inputs": [ + { + "id": "exitPort1PE", + "schema_ref": "schema3", + "app_data": { + "ui_data": { + "label": "exitPort1PE" + } + }, + "links": [ + { + "id": "a8417cbb-82a6-4d46-a56e-0de1f0460b5c", + "node_id_ref": "nodeID2PE", + "app_data": { + "ui_data": { + "class_name": "d3-data-link" + } + } + } + ] + } + ] + }, + { + "id": "exitID2PE", + "type": "binding", + "app_data": { + "ui_data": { + "label": "Save 2", + "x_pos": 904.8226928710938, + "y_pos": 225.94320678710938 + } + }, + "inputs": [ + { + "id": "exitPort2PE", + "schema_ref": "schema3", + "app_data": { + "ui_data": { + "label": "exitID2PE" + } + }, + "links": [ + { + "id": "0ff622eb-e345-4bf3-8459-eeb9eee7b9b2", + "node_id_ref": "nodeID3PE", + "app_data": { + "ui_data": { + "class_name": "d3-data-link" + } + } + } + ] + } + ] + } + ], + "app_data": { + "ui_data": { + "zoom": { + "x": 0, + "y": 0, + "k": 1 + }, + "comments": [ + { + "id": "4e32ba0a-b234-3e33-b321-d693bf8f8452", + "x_pos": 232, + "y_pos": 348, + "width": 197.35403442382812, + "height": 58.010650634765625, + "content": "This is a comment indicating the Supernode. The Supernode contains the sub-flow.", + "associated_id_refs": [ + { + "id": "980339b0-64b5-40b2-85fb-3302ab6e53e8", + "node_ref": "nodeIDSuperNodePE" + } + ] + } + ] + } + }, + "runtime_ref": "scala-spark-2.0.1" + }, + { + "id": "modeler-sub-pipeline", + "nodes": [ + { + "id": "entryID1SE", + "type": "binding", + "app_data": { + "ui_data": { + "label": "Binding 1", + "x_pos": 10, + "y_pos": 10 + } + }, + "outputs": [ + { + "id": "entryPort1SE", + "schema_ref": "schema3", + "app_data": { + "ui_data": { + "label": "entryPort1SE" + } + } + } + ] + }, + { + "id": "entryID2SE", + "type": "binding", + "app_data": { + "ui_data": { + "label": "Binding 2", + "x_pos": 10, + "y_pos": 10 + } + }, + "outputs": [ + { + "id": "entryPort2SE", + "schema_ref": "schema2", + "app_data": { + "ui_data": { + "label": "entryPort2SE" + } + } + } + ] + }, + { + "id": "nodeID1SE", + "type": "execution_node", + "op": "org.apache.spark.ml.ibm.transformers.Filter", + "app_data": { + "wml_data": { + "typeId": "filter-records" + }, + "ui_data": { + "label": "Filter", + "image": "filter-rows.svg", + "x_pos": 879.9398803710938, + "y_pos": -62.2780704498291, + "description": "Filter record data" + } + }, + "inputs": [ + { + "id": "input1nodeID1SE", + "schema_ref": "schema3", + "app_data": { + "ui_data": { + "class_name": "port-class", + "label": "input1nodeID1SE" + } + }, + "links": [ + { + "id": "5e0d1299-6315-4d2a-8adb-4f8e549b2bf0", + "node_id_ref": "entryID1SE" + } + ] + } + ], + "outputs": [ + { + "id": "output1nodeID1SE", + "schema_ref": "schema3", + "app_data": { + "ui_data": { + "class_name": "port-class", + "label": "output1nodeID1SE" + } + } + } + ], + "parameters": { + "conditionExpr": "Age > ${age}" + } + }, + { + "id": "nodeID2SE", + "type": "execution_node", + "op": "org.apache.spark.ml.ibm.transformers.Join", + "app_data": { + "wml_data": { + "typeId": "Join" + }, + "ui_data": { + "label": "Join", + "image": "join.svg", + "x_pos": 929.9966430664062, + "y_pos": 84.91569519042969, + "description": "Join two or more data records" + } + }, + "inputs": [ + { + "id": "input1NodeID2SE", + "schema_ref": "schema3", + "app_data": { + "ui_data": { + "cardinality": { + "min": 1, + "max": 1 + }, + "label": "input1NodeID2SE" + } + }, + "links": [ + { + "id": "c5127765-f412-4755-a633-56d7e8b32d0c", + "node_id_ref": "nodeID1SE" + } + ] + }, + { + "id": "input2NodeID2SE", + "schema_ref": "schema2", + "app_data": { + "ui_data": { + "cardinality": { + "min": 1, + "max": 1 + }, + "label": "input2NodeID2SE" + } + }, + "links": [ + { + "id": "db60a714-c295-4a3f-abf4-4f65fc45bd21", + "node_id_ref": "entryID2SE" + } + ] + } + ], + "outputs": [ + { + "id": "output1NodeID2SE", + "schema_ref": "schema2", + "app_data": { + "ui_data": { + "label": "output1NodeID2SE" + } + } + } + ], + "parameters": { + "join-type": "inner", + "key1": "inputDS1.NAME", + "key2": "inputDS2.name" + } + }, + { + "id": "exitID1SE", + "type": "binding", + "app_data": { + "ui_data": { + "label": "Binding 3", + "x_pos": 10, + "y_pos": 10 + } + }, + "inputs": [ + { + "id": "exitPort1SE", + "schema_ref": "schema2", + "app_data": { + "ui_data": { + "label": "" + } + }, + "links": [ + { + "id": "be8d3c92-6ee2-4973-9dbf-e91dd3c23b59", + "node_id_ref": "nodeID2SE" + } + ] + } + ] + } + ], + "app_data": { + "ui_data": { + "zoom": { + "x": 0, + "y": 0, + "k": 1.5 + }, + "comments": [] + } + }, + "runtime_ref": "SPSS-Modeler" + } + ], + "schemas": [ + { + "id": "schema1", + "fields": [ + { + "name": "integer2", + "type": "integer", + "nullable": false, + "metadata": { + "description": "Field integer2 is an integer field", + "measure": "range", + "modeling_role": "input", + "range": { + "min": -1, + "max": 400 + } + } + }, + { + "name": "double4", + "type": "double", + "nullable": false, + "metadata": { + "description": "Field double4 is a double field", + "measure": "range", + "modeling_role": "input", + "range": { + "min": 0.101, + "max": 100.211 + } + } + } + ] + }, + { + "id": "schema2", + "fields": [ + { + "name": "string1", + "type": "string", + "nullable": true, + "metadata": { + "description": "Field string1 is a string field", + "measure": "set", + "modeling_role": "input", + "max_length": 24, + "values": [ + "One", + "Two", + "Three" + ] + } + } + ] + }, + { + "id": "schema3", + "fields": [ + { + "name": "integer2", + "type": "integer", + "nullable": false, + "metadata": { + "description": "Field integer2 is an integer field", + "measure": "range", + "modeling_role": "input", + "range": { + "min": -1, + "max": 400 + } + } + }, + { + "name": "double4", + "type": "double", + "nullable": false, + "metadata": { + "description": "Field double4 is a double field", + "measure": "range", + "modeling_role": "input", + "range": { + "min": 0.101, + "max": 100.211 + } + } + }, + { + "name": "string1", + "type": "string", + "nullable": true, + "metadata": { + "description": "Field string1 is a string field", + "measure": "set", + "modeling_role": "input", + "max_length": 24, + "values": [ + "One", + "Two", + "Three" + ] + } + } + ] + } + ], + "runtimes": [ + { + "id": "scala-spark-2.0.1", + "name": "Scala-Spark", + "version": "2.0.1" + }, + { + "id": "SPSS-Modeler", + "name": "SPSS Modeler", + "version": "18.0" + } + ] + } diff --git a/common-pipeline/pipeline-flow/examples/pipeline-flow-v3-external-subflow-example.json b/common-pipeline/pipeline-flow/examples/pipeline-flow-v3-external-subflow-example.json new file mode 100644 index 0000000..f709010 --- /dev/null +++ b/common-pipeline/pipeline-flow/examples/pipeline-flow-v3-external-subflow-example.json @@ -0,0 +1,214 @@ +{ + "doc_type": "pipeline", + "version": "3.0", + "id": "abdc9881-f83a-4d38-b15b-8bf541e100fb", + "app_data": { + "ui_data": { + "name": "External Sub-Flow Example", + "description": "An example using a supernode that refers to an external sub-pipeline", + "class_name": "pipeline-flow-overall" + }, + "wml_data": { + "typeId": "stream" + } + }, + "primary_pipeline": "external-sub-flow-pipeline", + "pipelines": [ + { + "id": "external-sub-flow-pipeline", + "nodes": [ + { + "id": "entryID1", + "type": "binding", + "app_data": { + "ui_data": { + "label": "", + "x_pos": 10, + "y_pos": 10 + } + }, + "outputs": [ + { + "id": "entryPort1PE", + "schema_ref": "schema1", + "app_data": { + "ui_data": { + "label": "entryPort1PE" + } + } + } + ] + }, + { + "id": "superNode1", + "type": "super_node", + "subflow_ref": { + "pipeline_id_ref": "simple-pipeline", + "url": "https://github.com/elyra-ai/pipeline-schemas/blob/main/common-pipeline/pipeline-flow/examples/pipeline-flow-v2-example-simple.json" + }, + "app_data": { + "ui_data": { + "label": "", + "x_pos": 229, + "y_pos": 88, + "is_expanded": false, + "expanded_width": 200, + "expanded_height": 200 + } + }, + "inputs": [ + { + "id": "input1SuperNodePE", + "subflow_node_ref": "entryID1PE", + "schema_ref": "schema1", + "app_data": { + "ui_data": { + "class_name": "port-class", + "label": "Input Port" + } + }, + "links": [ + { + "id": "18240615-f293-47d0-92f2-9b7aa0ad5e67", + "node_id_ref": "entryID1" + } + ] + } + ], + "outputs": [ + { + "id": "output1SuperNodePE", + "subflow_node_ref": "exitID1PE", + "schema_ref": "schema2", + "app_data": { + "ui_data": { + "class_name": "port-class", + "label": "Output port" + } + } + } + ] + }, + { + "id": "exitID1", + "type": "binding", + "app_data": { + "ui_data": { + "label": "", + "x_pos": 446, + "y_pos": 69 + } + }, + "inputs": [ + { + "id": "exitPort1", + "schema_ref": "schema2", + "app_data": { + "ui_data": { + "label": "" + } + }, + "links": [ + { + "id": "57d3eab8-e4be-4f93-ac5e-dfe8e43923e7", + "node_id_ref": "superNode1" + } + ] + } + ] + } + ], + "app_data": { + "ui_data": { + "zoom": { + "x": 0, + "y": 0, + "k": 1 + }, + "comments": [] + } + }, + "runtime_ref": "scala-spark-2.0.1" + } + ], + "schemas": [ + { + "id": "schema1", + "fields": [ + { + "name": "integer2", + "type": "integer", + "nullable": false, + "metadata": { + "description": "Field integer2 is an integer field", + "measure": "range", + "modeling_role": "input", + "range": { + "min": -1, + "max": 400 + } + } + }, + { + "name": "double4", + "type": "double", + "nullable": false, + "metadata": { + "description": "Field double4 is a double field", + "measure": "range", + "modeling_role": "input", + "range": { + "min": 0.101, + "max": 100.211 + } + } + }, + { + "name": "string1", + "type": "string", + "nullable": true, + "metadata": { + "description": "Field string1 is a string field", + "measure": "set", + "modeling_role": "input", + "max_length": 24, + "values": [ + "One", + "Two", + "Three" + ] + } + } + ] + }, + { + "id": "schema2", + "fields": [ + { + "name": "string1", + "type": "string", + "nullable": true, + "metadata": { + "description": "Field string1 is a string field", + "measure": "set", + "modeling_role": "input", + "max_length": 24, + "values": [ + "One", + "Two", + "Three" + ] + } + } + ] + } + ], + "runtimes": [ + { + "id": "scala-spark-2.0.1", + "name": "Scala-Spark", + "version": "2.0.1" + } + ], + "json_schema": "https://api.dataplatform.ibm.com/schemas/common-pipeline/pipeline-flow/pipeline-flow-v3-schema.json" + } diff --git a/common-pipeline/pipeline-flow/examples/pipeline-flow-v3-modeling-example.json b/common-pipeline/pipeline-flow/examples/pipeline-flow-v3-modeling-example.json new file mode 100644 index 0000000..03fe400 --- /dev/null +++ b/common-pipeline/pipeline-flow/examples/pipeline-flow-v3-modeling-example.json @@ -0,0 +1,349 @@ +{ + "doc_type": "pipeline", + "version": "3.0", + "id": "e456f213-f83a-4d38-b15b-8bf541e100fb", + "app_data": { + "ui_data": { + "name": "Modeling Pipeline Flow Example", + "description": "An example ML pipeline - provides load/filter/model-build with an already-built model", + "class_name": "pipeline-flow-overall" + }, + "wml_data": { + "typeId": "stream" + } + }, + "primary_pipeline": "modeling-pipeline", + "pipelines": [ + { + "id": "modeling-pipeline", + "nodes": [ + { + "id": "entryID1PE", + "type": "binding", + "app_data": { + "ui_data": { + "label": "entryID1PE", + "x_pos": 10, + "y_pos": 10 + } + }, + "outputs": [ + { + "id": "entryPort1PE", + "schema_ref": "schema1", + "app_data": { + "ui_data": { + "label": "Output Port" + } + } + } + ] + }, + { + "id": "filter_nodeID2PE", + "type": "execution_node", + "op": "org.apache.spark.ml.ibm.transformers.Filter", + "app_data": { + "wml_data": { + "typeId": "filter-records" + }, + "ui_data": { + "label": "Filter", + "image": "", + "x_pos": 66, + "y_pos": 144, + "description": "Filter record data" + } + }, + "inputs": [ + { + "id": "input1NodeID2PE", + "schema_ref": "schema2", + "app_data": { + "ui_data": { + "cardinality": { + "min": 1, + "max": 1 + }, + "class_name": "port-class", + "label": "Input Port" + } + } + } + ], + "outputs": [ + { + "id": "output1NodeID2PE", + "schema_ref": "schema1", + "app_data": { + "ui_data": { + "class_name": "port-class", + "label": "Output port" + } + } + } + ], + "parameters": { + "conditionExpr": "integer2 > 20" + } + }, + { + "id": "modeling_nodeID3PE", + "type": "execution_node", + "op": "org.apache.spark.ml.regression.LinearRegression", + "app_data": { + "wml_data": { + "typeId": "linearregression", + "associations": [ + { + "ui_ref": "link_model_nodeID4PE", + "node_ref": "model_nodeID4PE", + "type": "trains_model" + } + ] + }, + "ui_data": { + "label": "Linear Regression", + "image": "", + "x_pos": 405, + "y_pos": 187, + "description": "Fit a Linear Regression Model", + "associations": [ + { + "id": "link_model_nodeID4PE", + "node_ref": "model_nodeID4PE" + } + ] + } + }, + "inputs": [ + { + "id": "input1modeling_nodeID3PE", + "schema_ref": "schema1", + "app_data": { + "ui_data": { + "class_name": "portClass", + "label": "input1modeling_nodeID3PE" + } + }, + "links": [ + { + "id": "4ec32b28-6997-4a2d-9906-1921b925fea5", + "node_id_ref": "filter_nodeID2PE" + } + ] + } + ], + "outputs": [ + { + "id": "output1modeling_nodeID3PE", + "schema_ref": "schema1", + "app_data": { + "ui_data": { + "class_name": "portClass", + "label": "output1modeling_nodeID3PE" + } + } + } + ], + "parameters": { + "targetField": [ + "integer2" + ], + "inputFieldList": [ + "double4", + "string1" + ], + "weightCol": [], + "elasticNetParam": 0, + "fitIntercept": false, + "maxIter": 90, + "regParam": 0.1, + "solver": "auto", + "standardization": true, + "tol": 0.000001 + } + }, + { + "id": "model_nodeID4PE", + "type": "model_node", + "model_ref": "https://reference.to.model.blob.in.storage", + "app_data": { + "wml_data": { + "typeId": "linearregressionmodel" + }, + "ui_data": { + "label": "Linear Regression Model", + "image": "", + "x_pos": 273, + "y_pos": 35, + "description": "Fitted Linear Regression Model" + } + }, + "inputs": [ + { + "id": "input1NodeID4PE", + "schema_ref": "schema1", + "app_data": { + "ui_data": { + "label": "input1NodeID4PE" + } + }, + "links": [ + { + "id": "de6724a4-c630-4f35-92dc-c8532eea5a06", + "node_id_ref": "entryID1PE" + } + ] + } + ], + "outputs": [ + { + "id": "output1NodeID4PE", + "schema_ref": "schema2", + "app_data": { + "ui_data": { + "class_name": "port-class", + "label": "Output port" + } + } + } + ] + } + ], + "app_data": { + "ui_data": { + "zoom": { + "x": 0, + "y": 0, + "k": 1 + }, + "comments": [] + } + }, + "runtime_ref": "scala-spark-2.0.1" + } + ], + "schemas": [ + { + "id": "schema1", + "fields": [ + { + "name": "integer2", + "type": "integer", + "nullable": false, + "metadata": { + "description": "Field integer2 is an integer field", + "measure": "range", + "modeling_role": "input", + "range": { + "min": -1, + "max": 400 + } + } + }, + { + "name": "double4", + "type": "double", + "nullable": false, + "metadata": { + "description": "Field double4 is a double field", + "measure": "range", + "modeling_role": "input", + "range": { + "min": 0.101, + "max": 100.211 + } + } + }, + { + "name": "string1", + "type": "string", + "nullable": true, + "metadata": { + "description": "Field string1 is a string field", + "measure": "set", + "modeling_role": "input", + "max_length": 24, + "values": [ + "One", + "Two", + "Three" + ] + } + } + ] + }, + { + "id": "schema2", + "fields": [ + { + "name": "integer2", + "type": "integer", + "nullable": false, + "metadata": { + "description": "Field integer2 is an integer field", + "measure": "range", + "modeling_role": "input", + "range": { + "min": -1, + "max": 400 + } + } + }, + { + "name": "double4", + "type": "double", + "nullable": false, + "metadata": { + "description": "Field double4 is a double field", + "measure": "range", + "modeling_role": "input", + "range": { + "min": 0.101, + "max": 100.211 + } + } + }, + { + "name": "string1", + "type": "string", + "nullable": true, + "metadata": { + "description": "Field string1 is a string field", + "measure": "set", + "modeling_role": "input", + "max_length": 24, + "values": [ + "One", + "Two", + "Three" + ] + } + }, + { + "name": "$integer2_predicted", + "type": "integer", + "nullable": false, + "metadata": { + "description": "Predicted field for 'integer2'", + "measure": "range", + "modeling_role": "input", + "range": { + "min": -1, + "max": 400 + } + } + } + ] + } + ], + "runtimes": [ + { + "id": "scala-spark-2.0.1", + "name": "Scala-Spark", + "version": "2.0.1" + } + ], + "json_schema": "https://api.dataplatform.ibm.com/schemas/common-pipeline/pipeline-flow/pipeline-flow-v3-schema.json" + }