Map itemprop meta tags to JSON-LD.

README.md

@@ -135,6 +135,27 @@ Fetch the first N bytes of a URL. Protection against too large return body.
 
 Fetch URL, extract metadata like mime type, title, description, images etc.
 
+The returned metadata can be queried with `z_url_metadata:p/2`, for example:
+
+    {ok, Metadata} = z_url_metadata:fetch(Url),
+    Title = z_url_metadata:p(title, Metadata),
+    Image = z_url_metadata:p(image, Metadata).
+
+JSON-LD data is available through the `json_ld` property:
+
+    JsonLDs = z_url_metadata:p(json_ld, Metadata).
+
+This list contains decoded JSON-LD maps from:
+
+ * `<script type="application/ld+json">` blocks.
+ * HTML microdata using `itemscope`, `itemtype`, `itemid`, and `itemprop`.
+
+Microdata is normalized to JSON-LD-style maps. Schema.org item types use
+`<<"https://schema.org">>` as `<<"@context">>` and the local type name as
+`<<"@type">>`. Scoped items with an `itemid` are added as separate maps in
+the list; scoped items without an `itemid` are embedded as property values of
+their parent item. Repeated properties are returned as lists.
+
 
 Tests
 -----

include/z_url_metadata.hrl

@@ -26,5 +26,6 @@
     is_index_page = false :: boolean(),
     headers :: list({binary(), binary()}),
     links :: #{ binary() => [ map() ] },
+    json_ld = [] :: [ map() ],
     partial_data :: binary()
 }).

rebar.config

@@ -11,6 +11,7 @@
 
 {deps, [
     {cowlib, "~> 2.11"},
+    {jsxrecord, "~> 2.0"},
     {qdate_localtime, "~> 1.2"},
     {tls_certificate_check, "~> 1.11"}
 ]}.

rebar.lock

@@ -1,5 +1,8 @@
 {"1.2.0",
 [{<<"cowlib">>,{pkg,<<"cowlib">>,<<"2.16.1">>},0},
+ {<<"euneus">>,{pkg,<<"euneus">>,<<"2.4.0">>},1},
+ {<<"json_polyfill">>,{pkg,<<"json_polyfill">>,<<"0.1.4">>},1},
+ {<<"jsxrecord">>,{pkg,<<"jsxrecord">>,<<"2.2.0">>},0},
  {<<"qdate_localtime">>,{pkg,<<"qdate_localtime">>,<<"1.2.2">>},0},
  {<<"ssl_verify_fun">>,{pkg,<<"ssl_verify_fun">>,<<"1.1.7">>},1},
  {<<"tls_certificate_check">>,
@@ -8,11 +11,17 @@
 [
 {pkg_hash,[
  {<<"cowlib">>, <<"318D385D55F657E9A5005838C4E426E13DCD724A691438384B6165A69687E531">>},
+ {<<"euneus">>, <<"E58178FB3B2C6FE139C863C4B347574E4C599BF1B9530ED3BAE7A10E2C7F870A">>},
+ {<<"json_polyfill">>, <<"ED9AD7A8CBDB8D1F1E59E22CDAE23A153A5BB93B5529AEBAEB189CA4B4C536C8">>},
+ {<<"jsxrecord">>, <<"486617130EAC71E5134CAF7CD4661A1850AE74B8BD628BDB586754D70A0FFA94">>},
  {<<"qdate_localtime">>, <<"43E1B20102F50A8B2A2BE7042C2F6BE989AD96CA2CC319DB5DF56E122E8873F6">>},
  {<<"ssl_verify_fun">>, <<"354C321CF377240C7B8716899E182CE4890C5938111A1296ADD3EC74CF1715DF">>},
  {<<"tls_certificate_check">>, <<"2CD065C0170BF8E1308F28BF9F95D9025D8FA455D7B3F5F05D8D1F0746D069B0">>}]},
 {pkg_hash_ext,[
  {<<"cowlib">>, <<"58F1E425A9E04176F1D30E20116F57C4E90EF0E187552E9741C465BDF4044F70">>},
+ {<<"euneus">>, <<"200EEFCE320314831AD79D77F03B109BC035E0A282B6B71FBCD3A7982804184D">>},
+ {<<"json_polyfill">>, <<"48C397EE2547FA459EDE01A30EC0E85717ABED3010867A63EEAAC5F203274303">>},
+ {<<"jsxrecord">>, <<"7C8D6F6EE0E253EC6C8BC68825403089E7E22860111D5AA764FA793FF195AAD4">>},
  {<<"qdate_localtime">>, <<"A38D5F1C5AE14B22F471E442B262AECCAFB915B664C7C364443DC73179C50FDA">>},
  {<<"ssl_verify_fun">>, <<"FE4C190E8F37401D30167C8C405EDA19469F34577987C76DDE613E838BBC67F8">>},
  {<<"tls_certificate_check">>, <<"5BA7EFF49AD6D3FB992E2825F09745C2B601738710D950222DCCEA312E092F83">>}]}

src/z_html_parse.erl

@@ -1320,7 +1320,10 @@ is_html_attr(<<"http-equiv">>) -> true;
 is_html_attr(<<"icon">>) -> true;
 is_html_attr(<<"id">>) -> true;
 is_html_attr(<<"ismap">>) -> true;
+is_html_attr(<<"itemid">>) -> true;
 is_html_attr(<<"itemprop">>) -> true;
+is_html_attr(<<"itemscope">>) -> true;
+is_html_attr(<<"itemtype">>) -> true;
 % K
 is_html_attr(<<"keytype">>) -> true;
 is_html_attr(<<"kind">>) -> true;