Edit Json Easily Programatically

2021-05-14T06:58:28+01:00

Introduction

Often I had to manipulate the JSON/YAML files. I created a utility that can help make changes in JSON/YAML easily.

Get the code.

git clone https://github.com/spareslant/json_manipulation.git
cd json_manipulation

Python version used

$ python --version
Python 3.9.0

How to Run:

input_data_1.json contents

{
    "data": [
        {
            "actions": [
                {
                    "link": "http://www.facebook.com/X999/posts/Y999",
                    "name": "Comment"
                },
                {
                    "link": "http://www.facebook.com/X999/posts/Y999",
                    "name": "Like"
                }
            ],
            "created_time": "2010-08-02T21:27:44+0000",
            "from": {
                "id": "X12",
                "name": "Tom Brady"
            },
            "id": "X999_Y999",
            "message": "Looking forward to 2010!",
            "type": "status",
            "updated_time": "2010-08-02T21:27:44+0000"
        },
        {
            "actions": [
                {
                    "link": "http://www.facebook.com/X998/posts/Y998",
                    "name": "Comment"
                },
                {
                    "link": "http://www.facebook.com/X998/posts/Y998",
                    "name": "Like"
                }
            ],
            "created_time": "2010-08-02T21:27:44+0000",
            "from": {
                "id": "X18",
                "name": "Peyton Manning"
            },
            "id": "X998_Y998",
            "message": "Where's my contract?",
            "type": "status",
            "updated_time": "2010-08-02T21:27:44+0000"
        }
    ],
    "job": {
        "Fulltimejob": "Lawyer",
        "hobbyjob": [
            "Gardening",
            "InteriorDesign"
        ]
    },
    "name": "Anonymous"
}

Generate Flat JSON structure

./generate_flat_json.py input_data_1.json > input_data_1_flat.json

input_data_1_flat.json file contents

{
  "data/0/actions/0/link": "http://www.facebook.com/X999/posts/Y999",
  "data/0/actions/0/name": "Comment",
  "data/0/actions/1/link": "http://www.facebook.com/X999/posts/Y999",
  "data/0/actions/1/name": "Like",
  "data/0/created_time": "2010-08-02T21:27:44+0000",
  "data/0/from/id": "X12",
  "data/0/from/name": "Tom Brady",
  "data/0/id": "X999_Y999",
  "data/0/message": "Looking forward to 2010!",
  "data/0/type": "status",
  "data/0/updated_time": "2010-08-02T21:27:44+0000",
  "data/1/actions/0/link": "http://www.facebook.com/X998/posts/Y998",
  "data/1/actions/0/name": "Comment",
  "data/1/actions/1/link": "http://www.facebook.com/X998/posts/Y998",
  "data/1/actions/1/name": "Like",
  "data/1/created_time": "2010-08-02T21:27:44+0000",
  "data/1/from/id": "X18",
  "data/1/from/name": "Peyton Manning",
  "data/1/id": "X998_Y998",
  "data/1/message": "Where's my contract?",
  "data/1/type": "status",
  "data/1/updated_time": "2010-08-02T21:27:44+0000",
  "job/Fulltimejob": "Lawyer",
  "job/hobbyjob/0": "Gardening",
  "job/hobbyjob/1": "InteriorDesign",
  "name": "Anonymous"
}

Make changes to above file/conents. like delete unwanted lines or change or add new ones. Following is the diff

diff -b -u flat_input_data_1.json flat_input_data_1.json.new                                                                                     1 ↵
--- flat_input_data_1.json      2021-05-14 06:33:56.000000000 +0100
+++ flat_input_data_1.json.new  2021-05-14 06:32:50.000000000 +0100
@@ -4,16 +4,18 @@
   "data/0/actions/1/link": "http://www.facebook.com/X999/posts/Y999",
   "data/0/actions/1/name": "Like",
   "data/0/created_time": "2010-08-02T21:27:44+0000",
-  "data/0/from/id": "X12",
+  "data/0/from/id": "Y9999",
   "data/0/from/name": "Tom Brady",
   "data/0/id": "X999_Y999",
   "data/0/message": "Looking forward to 2010!",
+  "data/0/urgentmessage": "Not at the moment",
   "data/0/type": "status",
   "data/0/updated_time": "2010-08-02T21:27:44+0000",
   "data/1/actions/0/link": "http://www.facebook.com/X998/posts/Y998",
   "data/1/actions/0/name": "Comment",
   "data/1/actions/1/link": "http://www.facebook.com/X998/posts/Y998",
   "data/1/actions/1/name": "Like",
+  "data/1/actions/2/link": "bogus.com",
   "data/1/created_time": "2010-08-02T21:27:44+0000",
   "data/1/from/id": "X18",
   "data/1/from/name": "Peyton Manning",
@@ -22,7 +24,9 @@
   "data/1/type": "status",
   "data/1/updated_time": "2010-08-02T21:27:44+0000",
   "job/Fulltimejob": "Lawyer",
+  "job/anotherJob": "consultant",
   "job/hobbyjob/0": "Gardening",
   "job/hobbyjob/1": "InteriorDesign",
-  "name": "Anonymous"
+  "name": "Anonymous",
+  "function": "Programming"

Re-generate JSON from flat JSON

./recreate_json_from_flat.py flat_input_data_1.json.new | python -m json.tool --sort-keys > recreated_data_1.json

Note: Input file was also sorted with above -m json.tool. Sorting is not required at all. We are sorting above json so that diff can be viewed comfortably.

recreated_data_1.json file contents

{
    "data": [
        {
            "actions": [
                {
                    "link": "http://www.facebook.com/X999/posts/Y999",
                    "name": "Comment"
                },
                {
                    "link": "http://www.facebook.com/X999/posts/Y999",
                    "name": "Like"
                }
            ],
            "created_time": "2010-08-02T21:27:44+0000",
            "from": {
                "id": "Y9999",
                "name": "Tom Brady"
            },
            "id": "X999_Y999",
            "message": "Looking forward to 2010!",
            "type": "status",
            "updated_time": "2010-08-02T21:27:44+0000",
            "urgentmessage": "Not at the moment"
        },
        {
            "actions": [
                {
                    "link": "http://www.facebook.com/X998/posts/Y998",
                    "name": "Comment"
                },
                {
                    "link": "http://www.facebook.com/X998/posts/Y998",
                    "name": "Like"
                },
                {
                    "link": "bogus.com"
                }
            ],
            "created_time": "2010-08-02T21:27:44+0000",
            "from": {
                "id": "X18",
                "name": "Peyton Manning"
            },
            "id": "X998_Y998",
            "message": "Where's my contract?",
            "type": "status",
            "updated_time": "2010-08-02T21:27:44+0000"
        }
    ],
    "function": "Programming",
    "job": {
        "Fulltimejob": "Lawyer",
        "anotherJob": "consultant",
        "hobbyjob": [
            "Gardening",
            "InteriorDesign"
        ]
    },
    "name": "Anonymous"
}

Following is the diff between original input json file and newly created file

diff -b -u input_data_1.json recreated_data_1.json
--- input_data_1.json   2021-05-14 06:26:45.000000000 +0100
+++ recreated_data_1.json       2021-05-14 06:36:45.000000000 +0100
@@ -13,13 +13,14 @@
             ],
             "created_time": "2010-08-02T21:27:44+0000",
             "from": {
-                "id": "X12",
+                "id": "Y9999",
                 "name": "Tom Brady"
             },
             "id": "X999_Y999",
             "message": "Looking forward to 2010!",
             "type": "status",
-            "updated_time": "2010-08-02T21:27:44+0000"
+            "updated_time": "2010-08-02T21:27:44+0000",
+            "urgentmessage": "Not at the moment"
         },
         {
             "actions": [
@@ -30,6 +31,9 @@
                 {
                     "link": "http://www.facebook.com/X998/posts/Y998",
                     "name": "Like"
+                },
+                {
+                    "link": "bogus.com"
                 }
             ],
             "created_time": "2010-08-02T21:27:44+0000",
@@ -43,8 +47,10 @@
             "updated_time": "2010-08-02T21:27:44+0000"
         }
     ],
+    "function": "Programming",
     "job": {
         "Fulltimejob": "Lawyer",
+        "anotherJob": "consultant",
         "hobbyjob": [
             "Gardening",
             "InteriorDesign"

Recreated json file may have some null elements in it, depending on what changes were introduced in flat json. remove_null_from_json.py removes those null values if not required.

Json Full Paths From Root Node to Leaf

2020-03-16T22:09:29Z

Introduction:

Often there is a requirement to create full paths from root node to leaf node in JSON file. Following python program will generate these paths.

Sample JSON file sample_json_file.json is below:

{
   "data": [
      {
         "id": "X999_Y999",
         "from": {
            "name": "Tom Brady", "id": "X12"
         },
         "message": "Looking forward to 2010!",
         "actions": [
            {
               "name": "Comment",
               "link": "http://www.facebook.com/X999/posts/Y999"
            },
            {
               "name": "Like",
               "link": "http://www.facebook.com/X999/posts/Y999"
            }
         ],
         "type": "status",
         "created_time": "2010-08-02T21:27:44+0000",
         "updated_time": "2010-08-02T21:27:44+0000"
      },
      {
         "id": "X998_Y998",
         "from": {
            "name": "Peyton Manning", "id": "X18"
         },
         "message": "Where's my contract?",
         "actions": [
            {
               "name": "Comment",
               "link": "http://www.facebook.com/X998/posts/Y998"
            },
            {
               "name": "Like",
               "link": "http://www.facebook.com/X998/posts/Y998"
            }
         ],
         "type": "status",
         "created_time": "2010-08-02T21:27:44+0000",
         "updated_time": "2010-08-02T21:27:44+0000"
      }
   ],
  "name": "Anonymous",
  "job": { "hobbyjob": [ "Gardening", "InteriorDesign"], "Fulltimejob": "Lawyer" }
}

Python program to create full paths from above JSON file

#! /usr/bin/env python

# This programm will create paths from root nodes to leafnodes along with values from any json file or structure.

import json
import pprint


json_data = open('sample_json_file.json', 'r').read()
json_dict = json.loads(json_data)

stack = []
final_dict = {}

def do_walk(datadict):

    if isinstance(datadict, dict):
        for key, value in datadict.items():
            stack.append(key)
            #print("/".join(stack))
            if isinstance(value, dict) and len(value.keys()) == 0:
                final_dict["/".join(stack)] = "EMPTY_DICT"
            if isinstance(value, list) and len(value) == 0:
                final_dict["/".join(stack)] = 'EMPTY_LIST'
            if isinstance(value, dict):
                do_walk(value)
            if isinstance(value, list):
                do_walk(value)
            if isinstance(value, unicode):
                final_dict["/".join(stack)] = value
            stack.pop()

    if isinstance(datadict, list):
        n = 0
        for key in datadict:
            stack.append(str(n))
            n = n + 1
            if isinstance(key, dict):
                do_walk(key)
            if isinstance(key, list):
                do_walk(key)
            if isinstance(key, unicode):
                final_dict["/".join(stack)] = key
            stack.pop()


do_walk(json_dict)
pprint.pprint(final_dict)

Below is the result:

# python create_json_paths.py
{u'data/0/actions/0/link': u'http://www.facebook.com/X999/posts/Y999',
 u'data/0/actions/0/name': u'Comment',
 u'data/0/actions/1/link': u'http://www.facebook.com/X999/posts/Y999',
 u'data/0/actions/1/name': u'Like',
 u'data/0/created_time': u'2010-08-02T21:27:44+0000',
 u'data/0/from/id': u'X12',
 u'data/0/from/name': u'Tom Brady',
 u'data/0/id': u'X999_Y999',
 u'data/0/message': u'Looking forward to 2010!',
 u'data/0/type': u'status',
 u'data/0/updated_time': u'2010-08-02T21:27:44+0000',
 u'data/1/actions/0/link': u'http://www.facebook.com/X998/posts/Y998',
 u'data/1/actions/0/name': u'Comment',
 u'data/1/actions/1/link': u'http://www.facebook.com/X998/posts/Y998',
 u'data/1/actions/1/name': u'Like',
 u'data/1/created_time': u'2010-08-02T21:27:44+0000',
 u'data/1/from/id': u'X18',
 u'data/1/from/name': u'Peyton Manning',
 u'data/1/id': u'X998_Y998',
 u'data/1/message': u"Where's my contract?",
 u'data/1/type': u'status',
 u'data/1/updated_time': u'2010-08-02T21:27:44+0000',
 u'job/Fulltimejob': u'Lawyer',
 u'job/hobbyjob/0': u'Gardening',
 u'job/hobbyjob/1': u'InteriorDesign',
 u'name': u'Anonymous'}

rand 'tech snippets'.hash.next

Friday, 14 May 2021