用于机器学习的多个 JSON 文件到单个 CSV 文件

问题描述

我正在使用 Cuckoo SandBox(OpenSource) 进行恶意软件分析,我必须收集多个恶意软件样本并从 Cuckoo SandBox 收集日志进行分析,导出的结果格式为 JSON

这是内部 JSON 文件的示例

{
    "info": {
        "added": 1611103440.152102,"started": 1611103482.539012,"duration": 301,"ended": 1611103783.806942,"owner": null,"score": 10,"id": 2005961,"category": "file","git": {
            "head": "","fetch_head": ""
        },"monitor": null,"package": "exe","route": "internet","custom": null,"machine": {
            "status": "stopped","name": "win7x6427","label": "win7x6427","manager": "VirtualBox","started_on": "2021-01-20 00:44:46","shutdown_on": "2021-01-20 00:49:43"
        },"platform": "windows","version": "2.0.7","options": "procmemdump=yes,route=internet"
    },"signatures": [
        {
            "families": [],"description": "Yara rules detected for file","severity": 1,"ttp": {},"markcount": 2,"references": [],"marks": [
                {
                    "type": "generic","description": "(no description)","rule": "UPX"
                },{
                    "type": "generic","description": "The packer/protector section names/keywords","rule": "suspicIoUs_packer_section"
                }
            ],"name": "file_yara"
        },{
            "families": [],"description": "The binary likely contains encrypted or compressed data indicative of a packer","severity": 2,"ttp": {
             
                "detected": true,"version": "2014.3.20.0","result": "Trojan.Win32.Z.Reveton.154624.A","normalized": [
                    "Reveton"
                ],"update": "20191121"
            },"ZoneAlarm": {
                "detected": true,"version": "1.0","result": "HEUR:Trojan.Win32.Generic","normalized": [],"update": "20191122"
            },"Avast-Mobile": {
                "detected": false,"version": "191121-00","result": null,"GData": {
                "detected": true,"version": "A:25.24054B:26.16732","result": "Gen:Variant.Kazy.58090","normalized": [
                    "Kazy"
                ],"TACHYON": {
                "detected": false,"version": "2019-11-22.01","AhnLab-V3": {
                "detected": false,"version": "3.16.5.25880","Acronis": {
                "detected": true,"version": "1.1.1.58","result": "suspicIoUs","update": "20191119"
            },"McAfee": {
                "detected": true,"version": "6.0.6.653","result": "Artemis!0A0B51CC7C2F","normalized": [
                    "Artemis"
                ],"MAX": {
                "detected": true,"version": "2019.9.16.1","result": "malware (ai score=99)","normalized": [
                    "ai score=99"
                ],"VBA32": {
                "detected": true,"version": "4.2.0","result": "TrojanDropper.Injector","Malwarebytes": {
                "detected": false,"version": "2.1.1.1115","Zoner": {
                "detected": false,"version": "1.0.0.1","TrendMicro-HouseCall": {
                "detected": true,"version": "10.0.0.1040","result": "Ransom_Reveton.R002C0DGR19","normalized": [
                    "Reveton","R002C0DGR19"
                ],"Rising": {
                "detected": true,"version": "25.0.0.24","result": "Ransom.Reveton!8.F2 (TFE:6:BRxiiJii4oE)","BRxiiJii4oE"
                ],"Yandex": {
                "detected": true,"version": "5.5.2.24","result": "Trojan.DR.Injector!wWbS84Bh3Bc","normalized": [
                    "wWbS84Bh3Bc"
                ],"SentinelOne": {
                "detected": true,"version": "1.11.0.53","result": "DFI - MalicIoUs PE","normalized": [
                    "MalicIoUs PE"
                ],"update": "20191118"
            },"MaxSecure": {
                "detected": true,"result": "Trojan.Malware.3664188.susgen","normalized": [
                    "susgen"
                ],"Fortinet": {
                "detected": true,"version": "6.2.137.0","result": "W32/Kryptik.ABC!tr","normalized": [
                    "Kryptik"
                ],"AVG": {
                "detected": true,"version": "18.4.3895.0","result": "Win32:Trojan-gen","Panda": {
                "detected": true,"version": "4.6.4.2","result": "Generic Malware","Qihoo-360": {
                "detected": true,"version": "1.0.0.1120","result": "HEUR/Malware.QVM38.Gen","normalized": [
                    "QVM38"
                ],"update": "20191122"
            }
        },"scan_id": "a941d1c84776b29324af06e737e8360b705db59791fb4d7df992e83a97056f1d-1574397337","sha1": "61261bd3d84cf71113ccfd962be5c67f64c0e74a","resource": "0a0b51cc7c2f01c59e8b45b7a53686cb","response_code": 1,"scan_date": "2019-11-22 04:35:37","permalink": "https://www.virustotal.com/gui/file/a941d1c84776b29324af06e737e8360b705db59791fb4d7df992e83a97056f1d/detection/f-a941d1c84776b29324af06e737e8360b705db59791fb4d7df992e83a97056f1d-1574397337","summary": {
            "positives": 56,"scan_date": "2019-11-22 04:35:37"
        },"sha256": "a941d1c84776b29324af06e737e8360b705db59791fb4d7df992e83a97056f1d","normalized": [
            "Kazy","Unsafe","Reveton","malicIoUs","confidence","R002C0DGR19","ZedlaF","jmOfaWhSwoei","Attribute","HighConfidence","kssoo","Malware@#363we32kn1oa8","XPACK","Bredo","Zbot","RBRH","AGeneric","moderate confidence","Artemis","ai score=99","BRxiiJii4oE","wWbS84Bh3Bc","MalicIoUs PE","susgen","Kryptik","QVM38"
        ],"total": 68,"positives": 56,"verbose_msg": "Scan finished,information embedded","md5": "0a0b51cc7c2f01c59e8b45b7a53686cb"
    },"network": {
        "udp": [],"dns_servers": [],"http": [],"smtp": [],"tcp": [],"geoinfo": {},"mitm": [],"https_ex": [],"http_ex": [],"sorted_pcap_sha256": "fd327e7ea77c6cfdb4f02d1d6e7f1a1d8e7abec7a41be8521670e0f6c77059f0","icmp": [],"tls": [],"dead_hosts": [],"smtp_ex": [],"hosts": [],"pcap_sha256": "fa95fe883afbba244cac7d186ab3c5c0193caea6e7c99269b8c09e71f0eb189e","dns": [],"domains": [],"irc": []
    },"resubmit": {
        "resubmit_ids": []
    },"static": {
        "pdb_path": null,"pe_imports": [
            {
                "imports": [
                    {
                        "name": "LoadLibraryA","address": "0x4603bc"
                    },{
                        "name": "GetProcAddress","address": "0x4603c0"
                    },{
                        "name": "VirtualProtect","address": "0x4603c4"
                    },{
                        "name": "VirtualAlloc","address": "0x4603c8"
                    },{
                        "name": "VirtualFree","address": "0x4603cc"
                    }
                ],"dll": "KERNEL32.DLL"
            },{
                "imports": [
                    {
                        "name": "AddAce","address": "0x4603d4"
                    }
                ],"dll": "advapi32.dll"
            },{
                "imports": [
                    {
                        "name": "DragFinish","address": "0x4603dc"
                    }
                ],"dll": "shell32.dll"
            },{
                "imports": [
                    {
                        "name": "EndMenu","address": "0x4603e4"
                    }
                ],"dll": "user32.dll"
            }
        ],"peid_signatures": null,"keys": [],"signature": [],"pe_timestamp": "2011-07-19 10:43:17","pe_exports": [
            {
                "ordinal": 1,"name": "Cobble","address": "0x429b9d"
            },{
                "ordinal": 2,"name": "H9922","address": "0x4235f0"
            },{
                "ordinal": 3,"name": "Inky","address": "0x4283e6"
            },{
                "ordinal": 4,"name": "Mouth","address": "0x42a364"
            },{
                "ordinal": 5,"name": "Oscar","address": "0x4246e3"
            }
        ],"imported_dll_count": 4,"pe_imphash": "ef529f580a16886d2de3a03af4c5a9e1","pe_resources": [
            {
                "name": "RT_BITMAP","language": "LANG_ENGLISH","filetype": "data","sublanguage": "SUBLANG_ENGLISH_US","offset": "0x00056358","size": "0x00000188"
            },{
                "name": "RT_MENU","offset": "0x00057d78","size": "0x00000c4a"
            },{
                "name": "RT_DIALOG","offset": "0x00059034","size": "0x000003c4"
            },{
                "name": "RT_STRING","offset": "0x000594e8","size": "0x00000134"
            },{
                "name": "RT_ACCELERATOR","offset": "0x000598bc","size": "0x000000c0"
            },"size": "0x000000c0"
            }
        ],"pe_versioninfo": [],"pe_sections": [
            {
                "size_of_data": "0x00000000","virtual_address": "0x00001000","entropy": 0.0,"name": "UPX0","virtual_size": "0x00039000"
            },{
                "size_of_data": "0x00025200","virtual_address": "0x0003a000","entropy": 7.994612275024696,"name": "UPX1","virtual_size": "0x00026000"
            },{
                "size_of_data": "0x00000600","virtual_address": "0x00060000","entropy": 2.84728083771964,"name": ".rsrc","virtual_size": "0x00001000"
            }
        ]
    },"snort": {
        "alerts": []
    },"behavior": {
        "generic": [
            {
                "process_path": "C:\\Windows\\System32\\lsass.exe","process_name": "lsass.exe","pid": 504,"summary": {},"first_seen": 1611103441.3125,"ppid": 388
            }
        ],"processes": [
            {
                "process_path": "C:\\Windows\\System32\\lsass.exe","calls": [],"track": false,"command_line": "C:\\Windows\\system32\\lsass.exe","modules": [
                    {
                        "basename": "lsass.exe","imgsize": 49152,"baseaddr": "0xfff30000","filepath": "C:\\Windows\\system32\\lsass.exe"
                    },{
                        "basename": "ntdll.dll","imgsize": 1740800,"baseaddr": "0x777c0000","filepath": "C:\\Windows\\SYstem32\\ntdll.dll"
                    },{
                        "basename": "kernel32.dll","imgsize": 1175552,}
    }
}

有几个像上面这样的 JSON 文件,我想创建一个大数据集 (CSV) 以执行训练和测试集

这个方法我试过了,但是不行

#! /bin/bash
#
# save this file as test.sh
#
mkdir -p "CSV"
for i in /Users/a/Downloads/Cuckoo Results/DESTDIR/*.json; do
    cat $i.json | jq -r '.[] | {column1: .path.to.data,column2: .path.to.data} | [.[] | tostring] | @csv' > extract-$i.csv
    echo "converted $i of many json files..."
    ((i=i+1))
done

我没有找到任何答案,非常感谢任何帮助

解决方法

暂无找到可以解决该程序问题的有效方法,小编努力寻找整理中!

如果你已经找到好的解决方法,欢迎将解决方案带上本链接一起发送给小编。

小编邮箱:dio#foxmail.com (将#修改为@)