[ASIM] Create EmailEvent parser for ProofPointTAPMessagesDeliveredV2_CL and ProofPointTAPMessagesBlockedV2_CL

.script/tests/KqlvalidationsTests/CustomTables/ProofPointTAPMessagesBlockedV2_CL.json

@@ -0,0 +1,129 @@
+{
+  "Name": "ProofPointTAPMessagesBlockedV2_CL",
+  "Properties": [
+    {
+      "Name": "ccAddresses",
+      "Type": "Dynamic"
+    },
+    {
+      "Name": "cluster",
+      "Type": "String"
+    },
+    {
+      "Name": "completelyRewritten",
+      "Type": "Bool"
+    },
+    {
+      "Name": "fromAddress",
+      "Type": "Dynamic"
+    },
+    {
+      "Name": "GUID",
+      "Type": "String"
+    },
+    {
+      "Name": "headerFrom",
+      "Type": "String"
+    },
+    {
+      "Name": "headerReplyTo",
+      "Type": "String"
+    },
+    {
+      "Name": "id",
+      "Type": "String"
+    },
+    {
+      "Name": "impostorScore",
+      "Type": "Int"
+    },
+    {
+      "Name": "malwareScore",
+      "Type": "Int"
+    },
+    {
+      "Name": "messageID",
+      "Type": "String"
+    },
+    {
+      "Name": "messageParts",
+      "Type": "Dynamic"
+    },
+    {
+      "Name": "messageSize",
+      "Type": "Int"
+    },
+    {
+      "Name": "messageTime",
+      "Type": "DateTime"
+    },
+    {
+      "Name": "modulesRun",
+      "Type": "Dynamic"
+    },
+    {
+      "Name": "phishScore",
+      "Type": "Int"
+    },
+    {
+      "Name": "policyRoutes",
+      "Type": "Dynamic"
+    },
+    {
+      "Name": "QID",
+      "Type": "String"
+    },
+    {
+      "Name": "quarantineFolder",
+      "Type": "String"
+    },
+    {
+      "Name": "quarantineRule",
+      "Type": "String"
+    },
+    {
+      "Name": "recipient",
+      "Type": "Dynamic"
+    },
+    {
+      "Name": "replyToAddress",
+      "Type": "Dynamic"
+    },
+    {
+      "Name": "sender",
+      "Type": "String"
+    },
+    {
+      "Name": "senderIP",
+      "Type": "String"
+    },
+    {
+      "Name": "spamScore",
+      "Type": "Int"
+    },
+    {
+      "Name": "subject",
+      "Type": "String"
+    },
+    {
+      "Name": "threatsInfoMap",
+      "Type": "Dynamic"
+    },
+    {
+      "Name": "toAddresses",
+      "Type": "Dynamic"
+    },
+    {
+      "Name": "xmailer",
+      "Type": "Dynamic"
+    },
+    {
+      "Name": "TimeGenerated",
+      "Type": "DateTime"
+    },
+    {
+      "Name": "Type",
+      "Type": "String"
+    }
+  ]
+}

.script/tests/KqlvalidationsTests/CustomTables/ProofPointTAPMessagesDeliveredV2_CL.json

@@ -2,43 +2,127 @@
   "Name": "ProofPointTAPMessagesDeliveredV2_CL",
   "Properties": [
     {
-      "Name": "TimeGenerated",
-      "Type": "DateTime"
+      "Name": "ccAddresses",
+      "Type": "Dynamic"
     },
     {
-      "Name": "threatsInfoMap",
+      "Name": "cluster",
+      "Type": "String"
+    },
+    {
+      "Name": "completelyRewritten",
+      "Type": "Bool"
+    },
+    {
+      "Name": "fromAddress",
       "Type": "Dynamic"
     },
+    {
+      "Name": "GUID",
+      "Type": "String"
+    },
+    {
+      "Name": "headerFrom",
+      "Type": "String"
+    },
+    {
+      "Name": "headerReplyTo",
+      "Type": "String"
+    },
+    {
+      "Name": "id",
+      "Type": "String"
+    },
+    {
+      "Name": "impostorScore",
+      "Type": "Int"
+    },
+    {
+      "Name": "malwareScore",
+      "Type": "Int"
+    },
+    {
+      "Name": "messageID",
+      "Type": "String"
+    },
     {
       "Name": "messageParts",
       "Type": "Dynamic"
     },
     {
-      "Name": "sender",
+      "Name": "messageSize",
+      "Type": "Int"
+    },
+    {
+      "Name": "messageTime",
+      "Type": "DateTime"
+    },
+    {
+      "Name": "modulesRun",
+      "Type": "Dynamic"
+    },
+    {
+      "Name": "phishScore",
+      "Type": "Int"
+    },
+    {
+      "Name": "policyRoutes",
+      "Type": "Dynamic"
+    },
+    {
+      "Name": "QID",
       "Type": "String"
     },
     {
-      "Name": "senderIP",
+      "Name": "quarantineFolder",
       "Type": "String"
     },
     {
-      "Name": "recipient",
+      "Name": "quarantineRule",
       "Type": "String"
     },
     {
-      "Name": "subject",
+      "Name": "recipient",
+      "Type": "Dynamic"
+    },
+    {
+      "Name": "replyToAddress",
+      "Type": "Dynamic"
+    },
+    {
+      "Name": "sender",
       "Type": "String"
     },
     {
-      "Name": "threatType",
+      "Name": "senderIP",
       "Type": "String"
     },
     {
-      "Name": "classification",
+      "Name": "spamScore",
+      "Type": "Int"
+    },
+    {
+      "Name": "subject",
       "Type": "String"
     },
     {
-      "Name": "filename",
+      "Name": "threatsInfoMap",
+      "Type": "Dynamic"
+    },
+    {
+      "Name": "toAddresses",
+      "Type": "Dynamic"
+    },
+    {
+      "Name": "xmailer",
+      "Type": "Dynamic"
+    },
+    {
+      "Name": "TimeGenerated",
+      "Type": "DateTime"
+    },
+    {
+      "name": "Type",
       "Type": "String"
     }
   ]

Parsers/ASimEmailEvent/ARM/ASimEmailEvent/ASimEmailEvent.json

@@ -27,7 +27,7 @@
         "displayName": "Email Event ASIM parser",
         "category": "ASIM",
         "FunctionAlias": "ASimEmailEvent",
-        "query": "let DisabledParsers=materialize(_GetWatchlist('ASimDisabledParsers') | where SearchKey in ('Any', 'ExcludeASimEmailEvent') | extend SourceSpecificParser=column_ifexists('SourceSpecificParser','') | distinct SourceSpecificParser| where isnotempty(SourceSpecificParser));\nlet ASimBuiltInDisabled=toscalar('ExcludeASimEmailEvent' in (DisabledParsers) or 'Any' in (DisabledParsers));\nlet parser=(pack:bool=false){\nunion isfuzzy=true\n  vimEmailEventEmpty,\n  ASimEmailEventProofpointPODMessage(\n    disabled=ASimBuiltInDisabled or ('ExcludeASimEmailEventProofpointPODMessage' in (DisabledParsers)),\n    pack=pack),\n  ASimEmailEventProofpointPODMailLog(pack=pack, disabled=(ASimBuiltInDisabled or ('ExcludeASimEmailEventProofpointPODMailLog' in (DisabledParsers)))),\n  ASimEmailEventMicrosoftDefenderOffice365(disabled=(ASimBuiltInDisabled or ('ExcludeASimEmailEventMicrosoftDefenderOffice365' in (DisabledParsers))), pack=pack),\n  ASimEmailEventProofpointTAPClicksPermitted(\n    disabled=ASimBuiltInDisabled or ('ExcludeASimEmailEventProofpointTAPClicksPermitted' in (DisabledParsers)),\n    pack=pack)\n};\nparser (pack=pack)\n",
+        "query": "let DisabledParsers=materialize(_GetWatchlist('ASimDisabledParsers') | where SearchKey in ('Any', 'ExcludeASimEmailEvent') | extend SourceSpecificParser=column_ifexists('SourceSpecificParser','') | distinct SourceSpecificParser| where isnotempty(SourceSpecificParser));\nlet ASimBuiltInDisabled=toscalar('ExcludeASimEmailEvent' in (DisabledParsers) or 'Any' in (DisabledParsers));\nlet parser=(pack:bool=false){\nunion isfuzzy=true\n  vimEmailEventEmpty,\n  ASimEmailEventProofpointPODMessage(disabled=ASimBuiltInDisabled or ('ExcludeASimEmailEventProofpointPODMessage' in (DisabledParsers)), pack=pack),\n  ASimEmailEventProofpointPODMailLog(pack=pack, disabled=(ASimBuiltInDisabled or ('ExcludeASimEmailEventProofpointPODMailLog' in (DisabledParsers)))),\n  ASimEmailEventMicrosoftDefenderOffice365(disabled=(ASimBuiltInDisabled or ('ExcludeASimEmailEventMicrosoftDefenderOffice365' in (DisabledParsers))), pack=pack),\n  ASimEmailEventProofpointTAPDelivered(disabled=ASimBuiltInDisabled or ('ExcludeASimEmailEventProofpointTAPDelivered' in (DisabledParsers)), pack=pack),\n  ASimEmailEventProofpointTAPBlocked(disabled=ASimBuiltInDisabled or ('ExcludeASimEmailEventProofpointTAPBlocked' in (DisabledParsers)), pack=pack),\n  ASimEmailEventProofpointTAPClicksPermitted(disabled=ASimBuiltInDisabled or ('ExcludeASimEmailEventProofpointTAPClicksPermitted' in (DisabledParsers)), pack=pack)\n};\nparser (pack=pack)\n",
         "version": 1,
         "functionParameters": "pack:bool=False"
       }

Parsers/ASimEmailEvent/ARM/ASimEmailEventProofpointTAPBlocked/ASimEmailEventProofpointTAPBlocked.json

@@ -0,0 +1,36 @@
+{
+  "$schema": "https://schema.management.azure.com/schemas/2019-08-01/deploymentTemplate.json#",
+  "contentVersion": "1.0.0.0",
+  "parameters": {
+    "Workspace": {
+      "type": "string",
+      "metadata": {
+        "description": "The Microsoft Sentinel workspace into which the function will be deployed. Has to be in the selected Resource Group."
+      }
+    },
+    "WorkspaceRegion": {
+      "type": "string",
+      "defaultValue": "[resourceGroup().location]",
+      "metadata": {
+        "description": "The region of the selected workspace. The default value will use the Region selection above."
+      }
+    }
+  },
+  "resources": [
+    {
+      "type": "Microsoft.OperationalInsights/workspaces/savedSearches",
+      "apiVersion": "2020-08-01",
+      "name": "[concat(parameters('Workspace'), '/ASimEmailEventProofpointTAPBlocked')]",
+      "location": "[parameters('WorkspaceRegion')]",
+      "properties": {
+        "etag": "*",
+        "displayName": "Email Event ASIM parser for Proofpoint TAP blocked messages",
+        "category": "ASIM",
+        "FunctionAlias": "ASimEmailEventProofpointTAPBlocked",
+        "query": "let parser = (disabled:bool = false, pack:bool = false) {\n    let Address = (value:dynamic) {\n        let text = iff(gettype(value) == \"array\",\n            iff(array_length(value) == 1, tostring(value[0]), \"\"), tostring(value));\n        trim(\" \", iff(indexof(text, \"<\") >= 0, tostring(split(tostring(split(text, \"<\")[1]), \">\")[0]), text))\n    };\n    let Addresses = (value:dynamic) {\n        parse_json(replace_strings(tostring(coalesce(value, dynamic([]))),\n            dynamic([\"<\", \">\"]), dynamic([\"\", \"\"])))\n    };\n    ProofPointTAPMessagesBlockedV2_CL\n    | where not(disabled)\n    | extend\n        EmailFromAddress = coalesce(Address(fromAddress), Address(headerFrom)),\n        EmailReplyToAddress = coalesce(Address(replyToAddress), Address(headerReplyTo)),\n        EmailMailFromAddress = Address(sender),\n        EmailToRecipients = Addresses(toAddresses),\n        EmailCCRecipients = Addresses(ccAddresses),\n        EnvelopeRecipients = Addresses(recipient),\n        // Only a single threat can populate scalar fields without losing multi-threat context.\n        Threat = case(gettype(threatsInfoMap) == \"dictionary\", threatsInfoMap,\n            array_length(threatsInfoMap) == 1, threatsInfoMap[0], dynamic(null)),\n        Part = case(gettype(messageParts) == \"dictionary\", messageParts,\n            array_length(messageParts) == 1, messageParts[0], dynamic(null))\n    | extend\n        EventStartTime = coalesce(messageTime, TimeGenerated),\n        EventEndTime = coalesce(messageTime, TimeGenerated),\n        EventCount = int(1),\n        EventType = \"EmailDelivery\",\n        EventOriginalType = \"messagesBlocked\",\n        EventResult = \"Failure\",\n        EventSchema = \"EmailEvent\",\n        EventSchemaVersion = \"1.0.0\",\n        EventVendor = \"Proofpoint\",\n        EventProduct = \"TAP Messages\",\n        EventOriginalUid = coalesce(GUID, tostring(column_ifexists(\"id\", \"\"))),\n        Dvc = coalesce(cluster, \"Proofpoint TAP\"),\n        DvcAction = \"Quarantined\",\n        DvcEntityKey = \"\",\n        AdditionalEntities = dynamic([]),\n        EmailFromDomain = tostring(split(EmailFromAddress, \"@\")[1]),\n        EmailMailFromDomain = tostring(split(EmailMailFromAddress, \"@\")[1]),\n        EmailReplyToDomain = tostring(split(EmailReplyToAddress, \"@\")[1]),\n        EmailDirection = \"Unknown\",\n        EmailDeliveryLocation = \"Quarantine\",\n        EmailProtocolName = \"SMTP\",\n        EmailSizeBytes = tolong(messageSize),\n        Dst = iff(array_length(EnvelopeRecipients) == 1, tostring(EnvelopeRecipients[0]),\n            iff(array_length(EnvelopeRecipients) > 1, tostring(EnvelopeRecipients), \"\")),\n        ThreatOriginalCategory = tostring(Threat.classification),\n        ThreatId = tostring(Threat.threatID),\n        ThreatName = tostring(Threat.threat),\n        ThreatFirstReportedTime = todatetime(Threat.threatTime),\n        ThreatIsActive = case(tolower(tostring(Threat.threatStatus)) == \"active\", true,\n            tolower(tostring(Threat.threatStatus)) in (\"cleared\", \"falsepositive\"), false, bool(null)),\n        EventReportUrl = tostring(Threat.threatUrl),\n        // Message parts can contain bodies as well as attachments; do not count them as files.\n        EmailFiles = iff(tolower(tostring(Part.disposition)) == \"attached\", pack_array(Part), dynamic([])),\n        RuleName = quarantineRule,\n        AdditionalFields = iff(pack, bag_pack(\n            \"GUID\", GUID, \"messageID\", messageID, \"QID\", QID,\n            \"cluster\", cluster, \"completelyRewritten\", completelyRewritten,\n            \"fromAddress\", fromAddress, \"headerFrom\", headerFrom, \"headerReplyTo\", headerReplyTo,\n            \"replyToAddress\", replyToAddress, \"sender\", sender,\n            \"recipient\", recipient, \"toAddresses\", toAddresses, \"ccAddresses\", ccAddresses,\n            \"impostorScore\", impostorScore, \"malwareScore\", malwareScore,\n            \"phishScore\", phishScore, \"spamScore\", spamScore,\n            \"messageParts\", messageParts, \"threatsInfoMap\", threatsInfoMap,\n            \"messageTime\", messageTime, \"messageSize\", messageSize,\n            \"modulesRun\", modulesRun, \"policyRoutes\", policyRoutes,\n            \"quarantineFolder\", quarantineFolder, \"quarantineRule\", quarantineRule,\n            \"xmailer\", xmailer, \"TenantId\", column_ifexists(\"TenantId\", \"\"),\n            \"_ResourceId\", column_ifexists(\"_ResourceId\", \"\")), dynamic({}))\n    | extend\n        EventUid = EventOriginalUid,\n        SrcUserName = EmailMailFromAddress,\n        Src = senderIP,\n        ThreatCategory = case(tolower(ThreatOriginalCategory) == \"phish\", \"Phishing\",\n            tolower(ThreatOriginalCategory) == \"malware\", \"Malware\",\n            tolower(ThreatOriginalCategory) == \"spam\", \"Spam\",\n            tolower(ThreatOriginalCategory) == \"impostor\", \"Impersonation\",\n            tolower(ThreatOriginalCategory) == \"toad\", \"TOAD\", \"\"),\n        EventResultDetails = case(tolower(ThreatOriginalCategory) == \"phish\", \"PhishingDetected\",\n            tolower(ThreatOriginalCategory) == \"malware\", \"MalwareDetected\", \"Other\")\n    | project-rename EmailSubject = subject, SrcIpAddr = senderIP, EmailOriginalDeliveryLocation = quarantineFolder\n    | extend IpAddr = SrcIpAddr, User = SrcUserName, Rule = RuleName\n    | project\n        TimeGenerated, Type, EventStartTime, EventEndTime, EventCount, EventType, EventOriginalType,\n        EventResult, EventResultDetails, EventSchema, EventSchemaVersion, EventVendor, EventProduct,\n        EventOriginalUid, EventUid, EventReportUrl, Dvc, DvcAction, DvcEntityKey, AdditionalEntities,\n        EmailFromAddress, EmailFromDomain, EmailMailFromAddress, EmailMailFromDomain,\n        EmailReplyToAddress, EmailReplyToDomain, EmailToRecipients, EmailCCRecipients,\n        EmailSubject, EmailSizeBytes, EmailDirection, EmailDeliveryLocation, EmailOriginalDeliveryLocation,\n        EmailProtocolName, EmailFiles,\n        Dst, Src, SrcIpAddr, IpAddr, SrcUserName, User, RuleName, Rule,\n        ThreatId, ThreatName, ThreatCategory, ThreatOriginalCategory, ThreatFirstReportedTime,\n        ThreatIsActive, AdditionalFields\n};\nparser(disabled=disabled, pack=pack)\n",
+        "version": 1,
+        "functionParameters": "disabled:bool=False,pack:bool=False"
+      }
+    }
+  ]
+}

Parsers/ASimEmailEvent/ARM/ASimEmailEventProofpointTAPBlocked/README.md

@@ -0,0 +1,22 @@
+# Proofpoint TAP ASIM EmailEvent Normalization Parser
+
+ARM template for ASIM EmailEvent schema parser for Proofpoint TAP.
+
+Normalizes Proofpoint TAP messages-blocked events from ProofPointTAPMessagesBlockedV2_CL to ASIM EmailEvent.
+This parser was created using the agentic ASIM parser creation tool, powered by Skills in GitHub.
+
+
+The Advanced Security Information Model (ASIM) enables you to use and create source-agnostic content, simplifying your analysis of the data in your Microsoft Sentinel workspace.
+
+For more information, see:
+
+- [Normalization and the Advanced Security Information Model (ASIM)](https://aka.ms/AboutASIM)
+- [Deploy all of ASIM](https://aka.ms/DeployASIM)
+- [ASIM EmailEvent normalization schema reference](https://aka.ms/ASimEmailEventDoc)
+
+For the changelog, see:
+- [CHANGELOG](https://github.com/Azure/Azure-Sentinel/blob/master/Parsers/ASimEmailEvent/CHANGELOG/ASimEmailEventProofpointTAPBlocked.md)
+
+<br>
+
+[![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2FAzure-Sentinel%2Fmaster%2FParsers%2FASimEmailEvent%2FARM%2FASimEmailEventProofpointTAPBlocked%2FASimEmailEventProofpointTAPBlocked.json) [![Deploy to Azure Gov](https://aka.ms/deploytoazuregovernbutton)](https://portal.azure.us/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2FAzure-Sentinel%2Fmaster%2FParsers%2FASimEmailEvent%2FARM%2FASimEmailEventProofpointTAPBlocked%2FASimEmailEventProofpointTAPBlocked.json)