From efa55251635f9bdda0cefb19508c4f750069a05b Mon Sep 17 00:00:00 2001 From: Marc Worrell Date: Tue, 30 Jun 2026 21:29:35 +0200 Subject: [PATCH 1/3] Map itemprop meta tags to JSON-LD. --- include/z_url_metadata.hrl | 1 + src/z_html_parse.erl | 3 + src/z_url_metadata.erl | 300 +++++++++++++++++++++++++++++++++---- test/z_html_parse_test.erl | 14 ++ 4 files changed, 288 insertions(+), 30 deletions(-) diff --git a/include/z_url_metadata.hrl b/include/z_url_metadata.hrl index 018353b..ae86620 100644 --- a/include/z_url_metadata.hrl +++ b/include/z_url_metadata.hrl @@ -26,5 +26,6 @@ is_index_page = false :: boolean(), headers :: list({binary(), binary()}), links :: #{ binary() => [ map() ] }, + json_ld = [] :: [ map() ], partial_data :: binary() }). diff --git a/src/z_html_parse.erl b/src/z_html_parse.erl index c79cf82..c2c28af 100644 --- a/src/z_html_parse.erl +++ b/src/z_html_parse.erl @@ -1320,7 +1320,10 @@ is_html_attr(<<"http-equiv">>) -> true; is_html_attr(<<"icon">>) -> true; is_html_attr(<<"id">>) -> true; is_html_attr(<<"ismap">>) -> true; +is_html_attr(<<"itemid">>) -> true; is_html_attr(<<"itemprop">>) -> true; +is_html_attr(<<"itemscope">>) -> true; +is_html_attr(<<"itemtype">>) -> true; % K is_html_attr(<<"keytype">>) -> true; is_html_attr(<<"kind">>) -> true; diff --git a/src/z_url_metadata.erl b/src/z_url_metadata.erl index 90dfc8e..b72d1a1 100644 --- a/src/z_url_metadata.erl +++ b/src/z_url_metadata.erl @@ -47,7 +47,7 @@ -type metadata() :: #url_metadata{}. -type property() :: mime | mime_options | site_name | content_length | url | canonical_url | short_url | final_url | links | - headers | title | h1 | summary | tags | filename | + headers | title | h1 | summary | tags | filename | json_ld | mtitle | description | keywords | author | charset | language | image | image_nav | thumbnail | icon | icon_nav | icon_shortcut | icon_touch | @@ -126,9 +126,10 @@ fetch_data(BaseUrl, Hs, Data) -> -spec p(Property, Metadata) -> Value when Property :: property() | [ property() ], Metadata :: metadata(), - Value :: binary() | list( binary() ) | Headers | Links | undefined, + Value :: binary() | list( binary() ) | Headers | Links | JsonLDs | undefined, Headers :: list({binary(), binary()}), - Links :: #{binary() => [map()]}. + Links :: #{binary() => [map()]}, + JsonLDs :: [map()]. p(mime, MD) -> MD#url_metadata.content_type; p(mime_options, MD) -> @@ -165,6 +166,8 @@ p(headers, MD) -> MD#url_metadata.headers; p(links, MD) -> MD#url_metadata.links; +p(json_ld, MD) -> + MD#url_metadata.json_ld; p(title, MD) -> case p1([<<"og:title">>, <<"twitter:title">>, mtitle, h1, title], MD) of undefined -> p(filename, MD); @@ -370,7 +373,9 @@ partial_metadata(Url, Hs, Data) -> IsHTML = IsText andalso is_html(CT), Data1 = maybe_convert_utf8(IsText, IsHTML, proplists:get_value(<<"charset">>, CTOpts), Data), MetadataList = html_meta(IsHTML, Data1), - {LinkList, MetadataList1} = lists:partition(fun({P, _}) -> P =:= link end, MetadataList), + {JsonLDList, MetadataList0} = lists:partition(fun({P, _}) -> P =:= json_ld end, MetadataList), + {LinkList, MetadataList1} = lists:partition(fun({P, _}) -> P =:= link end, MetadataList0), + JsonLDs = lists:append([ LDs || {json_ld, LDs} <- JsonLDList ]), Links = lists:foldr( fun({link, {Rel, As}}, Acc) -> Acc#{ @@ -387,6 +392,7 @@ partial_metadata(Url, Hs, Data) -> content_length = content_length(HsBin), metadata = MetadataList1, links = Links1, + json_ld = JsonLDs, is_index_page = is_index_page(Url), headers = HsBin, partial_data = Data @@ -412,7 +418,12 @@ html_meta(Data) -> html_meta(true, PartialData) -> case parse(PartialData) of {ok, Parsed} -> - lists:reverse(html(Parsed, [], #ps{})); + JsonLDs = json_ld(Parsed), + Metadata = lists:reverse(html(Parsed, [], #ps{})), + case JsonLDs of + [] -> Metadata; + _ -> [{json_ld, JsonLDs} | Metadata] + end; {error, _} -> [] end; @@ -454,22 +465,14 @@ tag({<<"html">>, As, Es}, MD, P) -> tag({<<"meta">>, As, _}, MD, P) -> Name = z_string:to_lower(proplists:get_value(<<"name">>, As)), Property = proplists:get_value(<<"property">>, As), - ItemProp = proplists:get_value(<<"itemprop">>, As), HttpEquiv = proplists:get_value(<<"http-equiv">>, As), Value = proplists:get_value(<<"value">>, As), Content = proplists:get_value(<<"content">>, As, Value), case first([Name, Property, HttpEquiv]) of undefined -> - case ItemProp of - undefined -> - case proplists:get_value(<<"charset">>, As) of - undefined -> {MD, P}; - Charset -> {[{charset,Charset} | MD], P} - end; - <<>> -> - {MD, P}; - _ -> - {meta_itemprop_tag(z_string:to_lower(ItemProp), Content, MD), P} + case proplists:get_value(<<"charset">>, As) of + undefined -> {MD, P}; + Charset -> {[{charset,Charset} | MD], P} end; Prop -> {meta_tag(Prop, Content, MD), P} @@ -538,6 +541,7 @@ meta_tag(_Name, undefined, MD) -> MD; meta_tag(_Name, <<>>, MD) -> MD; meta_tag(<<"og:", _/binary>> = OG, Content, MD) -> [{OG, Content}|MD]; meta_tag(<<"twitter:", _/binary>> = Tw, Content, MD) -> [{Tw, Content}|MD]; +meta_tag(<<"al:", _/binary>> = Al, Content, MD) -> [{Al, Content}|MD]; meta_tag(<<"title">>, Content, MD) -> [{mtitle, Content}|MD]; meta_tag(<<"keywords">>, Content, MD) -> [{keywords, Content}|MD]; meta_tag(<<"description">>, Content, MD) -> [{description, Content}|MD]; @@ -545,15 +549,150 @@ meta_tag(<<"author">>, Content, MD) -> [{author, Content}|MD]; meta_tag(<<"thumbnail">>, Content, MD) -> [{thumbnail, Content}|MD]; meta_tag(<<"content-type">>, Content, MD) -> [{content_type, Content}|MD]; meta_tag(<<"duration">>, Content, MD) -> [{duration, Content}|MD]; -meta_tag(<<"datePublished">>, Content, MD) -> [{date_published, Content}|MD]; -meta_tag(<<"uploadDate">>, Content, MD) -> [{date_uploaded, Content}|MD]; -meta_tag(<<"embedUrl">>, Content, MD) -> [{embed_url, Content}|MD]; -meta_tag(<<"contentUrl">>, Content, MD) -> [{content_url, Content}|MD]; +meta_tag(<<"datepublished">>, Content, MD) -> [{date_published, Content}|MD]; +meta_tag(<<"uploaddate">>, Content, MD) -> [{date_uploaded, Content}|MD]; +meta_tag(<<"embedurl">>, Content, MD) -> [{embed_url, Content}|MD]; +meta_tag(<<"contenturl">>, Content, MD) -> [{content_url, Content}|MD]; meta_tag(_Name, _Content, MD) -> MD. -meta_itemprop_tag(<<"name">>, Content, MD) -> meta_tag(<<"title">>, Content, MD); -meta_itemprop_tag(<<"thumbnailurl">>, Content, MD) -> meta_tag(<<"thumbnail">>, Content, MD); -meta_itemprop_tag(Name, Content, MD) -> meta_tag(Name, Content, MD). +json_ld(Es) -> + json_ld(Es, []). + +json_ld([], Acc) -> + lists:reverse(Acc); +json_ld([E|Es], Acc) -> + json_ld(Es, json_ld(E, Acc)); +json_ld({comment, _}, Acc) -> + Acc; +json_ld({pi, _Xml, _Attrs}, Acc) -> + Acc; +json_ld({_, As, _} = Tag, Acc) -> + case proplists:is_defined(<<"itemscope">>, As) of + true -> + {_Object, Acc1} = json_ld_item(Tag, Acc), + Acc1; + false -> + json_ld_children(Tag, Acc) + end; +json_ld(_Text, Acc) -> + Acc. + +json_ld_children({_Tag, _As, Es}, Acc) -> + json_ld(Es, Acc). + +json_ld_item({_Tag, As, Es}, Acc) -> + Object0 = json_ld_item_base(As), + {Object, Acc1} = json_ld_props(Es, Object0, Acc), + Acc2 = case proplists:get_value(<<"itemid">>, As) of + undefined -> Acc1; + <<>> -> Acc1; + _ -> [Object | Acc1] + end, + {Object, Acc2}. + +json_ld_item_base(As) -> + Object0 = case normalize_itemtype(proplists:get_value(<<"itemtype">>, As)) of + undefined -> + #{}; + {schema, Context, Type} -> + #{ <<"@context">> => Context, <<"@type">> => Type }; + {type, Type} -> + #{ <<"@type">> => Type } + end, + case proplists:get_value(<<"itemid">>, As) of + undefined -> Object0; + <<>> -> Object0; + ItemId -> Object0#{ <<"@id">> => ItemId } + end. + +json_ld_props([], Object, Acc) -> + {Object, Acc}; +json_ld_props([E|Es], Object, Acc) -> + {Object1, Acc1} = json_ld_prop(E, Object, Acc), + json_ld_props(Es, Object1, Acc1); +json_ld_props(_Text, Object, Acc) -> + {Object, Acc}. + +json_ld_prop({comment, _}, Object, Acc) -> + {Object, Acc}; +json_ld_prop({pi, _Xml, _Attrs}, Object, Acc) -> + {Object, Acc}; +json_ld_prop({_Tag, As, _Es} = Tag, Object, Acc) -> + ItemProp = proplists:get_value(<<"itemprop">>, As), + HasScope = proplists:is_defined(<<"itemscope">>, As), + case {itemprop_names(ItemProp), HasScope} of + {[], true} -> + {_Nested, Acc1} = json_ld_item(Tag, Acc), + {Object, Acc1}; + {[], false} -> + json_ld_props_child(Tag, Object, Acc); + {Props, true} -> + {Nested, Acc1} = json_ld_item(Tag, Acc), + {json_ld_add_props(Props, Nested, Object), Acc1}; + {Props, false} -> + {json_ld_add_props(Props, itemprop_value(Tag), Object), Acc} + end; +json_ld_prop(_Text, Object, Acc) -> + {Object, Acc}. + +json_ld_props_child({_Tag, _As, Es}, Object, Acc) -> + json_ld_props(Es, Object, Acc). + +itemprop_names(undefined) -> + []; +itemprop_names(<<>>) -> + []; +itemprop_names(ItemProp) -> + [ Prop || Prop <- binary:split(ItemProp, <<" ">>, [global]), Prop =/= <<>> ]. + +json_ld_add_props([], _Value, Object) -> + Object; +json_ld_add_props([Prop|Props], Value, Object) -> + Object1 = case is_empty_itemprop_value(Value) of + true -> + Object; + false -> + Object#{ Prop => json_ld_append_value(maps:get(Prop, Object, undefined), Value) } + end, + json_ld_add_props(Props, Value, Object1). + +is_empty_itemprop_value(undefined) -> true; +is_empty_itemprop_value(<<>>) -> true; +is_empty_itemprop_value(_) -> false. + +json_ld_append_value(undefined, Value) -> + Value; +json_ld_append_value(Values, Value) when is_list(Values) -> + Values ++ [Value]; +json_ld_append_value(OldValue, Value) -> + [OldValue, Value]. + +itemprop_value({_Tag, As, Es}) -> + case first([ + proplists:get_value(<<"content">>, As), + proplists:get_value(<<"href">>, As), + proplists:get_value(<<"src">>, As), + proplists:get_value(<<"data">>, As), + proplists:get_value(<<"datetime">>, As) + ]) of + undefined -> z_string:trim(fetch_text(Es, <<>>)); + Value -> Value + end. + +normalize_itemtype(undefined) -> + undefined; +normalize_itemtype(<<>>) -> + undefined; +normalize_itemtype(ItemType) -> + Type = hd(binary:split(ItemType, <<" ">>, [global])), + normalize_schema_itemtype(Type). + +normalize_schema_itemtype(<<"http://schema.org/", Type/binary>>) -> + {schema, <<"https://schema.org">>, Type}; +normalize_schema_itemtype(<<"https://schema.org/", Type/binary>>) -> + {schema, <<"https://schema.org">>, Type}; +normalize_schema_itemtype(Type) -> + {type, Type}. meta_link(_Name, undefined, _As, MD) -> MD; meta_link(_Name, <<>>, _As, MD) -> MD; @@ -849,21 +988,122 @@ partial_ampersant_in_html_meta_test() -> youtube_itemprop_html_meta_test() -> Data = <<" - - - - +
+ + + + +
">>, MD = html_meta(Data), - ?assertEqual(<<"Example Video">>, proplists:get_value(mtitle, MD)), - ?assertEqual(<<"Example Description">>, proplists:get_value(description, MD)), + JsonLDs = proplists:get_value(json_ld, MD), + ?assertEqual(2, length(JsonLDs)), + Video = json_ld_by_id(<<"https://www.youtube.com/watch?v=example">>, JsonLDs), + Author = json_ld_by_id(<<"http://www.youtube.com/@example">>, JsonLDs), + ?assertEqual(<<"https://schema.org">>, maps:get(<<"@context">>, Video)), + ?assertEqual(<<"VideoObject">>, maps:get(<<"@type">>, Video)), + ?assertEqual(<<"Example Video">>, maps:get(<<"name">>, Video)), + ?assertEqual(<<"Example Description">>, maps:get(<<"description">>, Video)), ?assertEqual(<<"https://i.ytimg.com/vi/example/maxresdefault.jpg">>, - proplists:get_value(thumbnail, MD)), + maps:get(<<"thumbnailUrl">>, Video)), + ?assertEqual(Author, maps:get(<<"author">>, Video)), + ?assertEqual(<<"Person">>, maps:get(<<"@type">>, Author)), + ?assertEqual(<<"Example Channel">>, maps:get(<<"name">>, Author)), + ?assertEqual(<<"http://www.youtube.com/@example">>, maps:get(<<"url">>, Author)), + ?assertEqual(undefined, proplists:get_value(mtitle, MD)), + ok. + +youtube_itemprop_metadata_property_test() -> + Data = <<" +
+ +
+ ">>, + {ok, MD} = fetch_data([], Data), + [Video] = p(json_ld, MD), + ?assertEqual(<<"Example Video">>, maps:get(<<"name">>, Video)), + ?assertEqual(<<"VideoObject">>, maps:get(<<"@type">>, Video)), ok. +youtube_scoped_itemprop_html_meta_test() -> + Data = <<" +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+ +
+ + +
+ + + +
+ ">>, + JsonLDs = proplists:get_value(json_ld, html_meta(Data)), + ?assertEqual(2, length(JsonLDs)), + Video = json_ld_by_id(<<"https://www.youtube.com/watch?v=6neL1YuX6kQ">>, JsonLDs), + Channel = json_ld_by_id(<<"http://www.youtube.com/@WesODonnellX">>, JsonLDs), + Author = maps:get(<<"author">>, Video), + Thumbnail = maps:get(<<"thumbnail">>, Video), + InteractionStats = maps:get(<<"interactionStatistic">>, Video), + ?assertEqual(<<"VideoObject">>, maps:get(<<"@type">>, Video)), + ?assertEqual(<<"Europe's New Anti-Drone Cannon Is Spreading Across Six NATO Armies">>, + maps:get(<<"name">>, Video)), + ?assertEqual(<<"False">>, maps:get(<<"requiresSubscription">>, Video)), + ?assertEqual(<<"6neL1YuX6kQ">>, maps:get(<<"identifier">>, Video)), + ?assertEqual(<<"PT12M58S">>, maps:get(<<"duration">>, Video)), + ?assertEqual(<<"https://i.ytimg.com/vi/6neL1YuX6kQ/maxresdefault.jpg">>, + maps:get(<<"thumbnailUrl">>, Video)), + ?assertEqual(<<"https://www.youtube.com/embed/6neL1YuX6kQ">>, maps:get(<<"embedUrl">>, Video)), + ?assertEqual(<<"Nonprofits & Activism">>, maps:get(<<"genre">>, Video)), + ?assertEqual(<<"Person">>, maps:get(<<"@type">>, Author)), + ?assertEqual(<<"Wes O'Donnell">>, maps:get(<<"name">>, Author)), + ?assertEqual(<<"ImageObject">>, maps:get(<<"@type">>, Thumbnail)), + ?assertEqual(<<"1280">>, maps:get(<<"width">>, Thumbnail)), + ?assertMatch([_, _], InteractionStats), + ?assertEqual(<<"Thing">>, maps:get(<<"@type">>, Channel)), + ?assertEqual(<<"Wes O'Donnell">>, maps:get(<<"name">>, Channel)), + ok. + +json_ld_by_id(ItemId, JsonLDs) -> + hd([ JsonLD || JsonLD <- JsonLDs, maps:get(<<"@id">>, JsonLD, undefined) =:= ItemId ]). + links_header_test() -> Data = <<" diff --git a/test/z_html_parse_test.erl b/test/z_html_parse_test.erl index 615d71a..5ca898e 100644 --- a/test/z_html_parse_test.erl +++ b/test/z_html_parse_test.erl @@ -266,6 +266,20 @@ parse_test() -> ok. +parse_microdata_attrs_test() -> + D0 = <<"
">>, + ?assertEqual( + {ok, {<<"div">>, + [{<<"itemscope">>, <<"itemscope">>}, + {<<"itemtype">>, <<"https://schema.org/VideoObject">>}, + {<<"itemid">>, <<"video-1">>}], + [{<<"meta">>, + [{<<"itemprop">>, <<"name">>}, + {<<"content">>, <<"Example">>}], + []}]}}, + z_html_parse:parse(D0)), + ok. + exhaustive_is_singleton_test() -> T = z_cover:clause_lookup_table(z_html_parse, is_singleton), [?assertEqual(V, z_html_parse:is_singleton(K, #{ mode => html })) || {K, V} <- T]. From 4456fa11ad018356c6c9a74db249cad43b8db87c Mon Sep 17 00:00:00 2001 From: Marc Worrell Date: Tue, 30 Jun 2026 21:39:30 +0200 Subject: [PATCH 2/3] Add JSON-LD support, parse itemprop into JSON-LD --- rebar.config | 1 + rebar.lock | 9 ++++ src/z_url_metadata.erl | 92 ++++++++++++++++++++++++++++++++++++-- src/zotonic_stdlib.app.src | 2 +- 4 files changed, 99 insertions(+), 5 deletions(-) diff --git a/rebar.config b/rebar.config index 384481c..2474833 100644 --- a/rebar.config +++ b/rebar.config @@ -11,6 +11,7 @@ {deps, [ {cowlib, "~> 2.11"}, + {jsxrecord, "~> 2.0"}, {qdate_localtime, "~> 1.2"}, {tls_certificate_check, "~> 1.11"} ]}. diff --git a/rebar.lock b/rebar.lock index a0d0c1e..061e63e 100644 --- a/rebar.lock +++ b/rebar.lock @@ -1,5 +1,8 @@ {"1.2.0", [{<<"cowlib">>,{pkg,<<"cowlib">>,<<"2.16.1">>},0}, + {<<"euneus">>,{pkg,<<"euneus">>,<<"2.4.0">>},1}, + {<<"json_polyfill">>,{pkg,<<"json_polyfill">>,<<"0.1.4">>},1}, + {<<"jsxrecord">>,{pkg,<<"jsxrecord">>,<<"2.2.0">>},0}, {<<"qdate_localtime">>,{pkg,<<"qdate_localtime">>,<<"1.2.2">>},0}, {<<"ssl_verify_fun">>,{pkg,<<"ssl_verify_fun">>,<<"1.1.7">>},1}, {<<"tls_certificate_check">>, @@ -8,11 +11,17 @@ [ {pkg_hash,[ {<<"cowlib">>, <<"318D385D55F657E9A5005838C4E426E13DCD724A691438384B6165A69687E531">>}, + {<<"euneus">>, <<"E58178FB3B2C6FE139C863C4B347574E4C599BF1B9530ED3BAE7A10E2C7F870A">>}, + {<<"json_polyfill">>, <<"ED9AD7A8CBDB8D1F1E59E22CDAE23A153A5BB93B5529AEBAEB189CA4B4C536C8">>}, + {<<"jsxrecord">>, <<"486617130EAC71E5134CAF7CD4661A1850AE74B8BD628BDB586754D70A0FFA94">>}, {<<"qdate_localtime">>, <<"43E1B20102F50A8B2A2BE7042C2F6BE989AD96CA2CC319DB5DF56E122E8873F6">>}, {<<"ssl_verify_fun">>, <<"354C321CF377240C7B8716899E182CE4890C5938111A1296ADD3EC74CF1715DF">>}, {<<"tls_certificate_check">>, <<"2CD065C0170BF8E1308F28BF9F95D9025D8FA455D7B3F5F05D8D1F0746D069B0">>}]}, {pkg_hash_ext,[ {<<"cowlib">>, <<"58F1E425A9E04176F1D30E20116F57C4E90EF0E187552E9741C465BDF4044F70">>}, + {<<"euneus">>, <<"200EEFCE320314831AD79D77F03B109BC035E0A282B6B71FBCD3A7982804184D">>}, + {<<"json_polyfill">>, <<"48C397EE2547FA459EDE01A30EC0E85717ABED3010867A63EEAAC5F203274303">>}, + {<<"jsxrecord">>, <<"7C8D6F6EE0E253EC6C8BC68825403089E7E22860111D5AA764FA793FF195AAD4">>}, {<<"qdate_localtime">>, <<"A38D5F1C5AE14B22F471E442B262AECCAFB915B664C7C364443DC73179C50FDA">>}, {<<"ssl_verify_fun">>, <<"FE4C190E8F37401D30167C8C405EDA19469F34577987C76DDE613E838BBC67F8">>}, {<<"tls_certificate_check">>, <<"5BA7EFF49AD6D3FB992E2825F09745C2B601738710D950222DCCEA312E092F83">>}]} diff --git a/src/z_url_metadata.erl b/src/z_url_metadata.erl index b72d1a1..8036545 100644 --- a/src/z_url_metadata.erl +++ b/src/z_url_metadata.erl @@ -566,16 +566,26 @@ json_ld({comment, _}, Acc) -> Acc; json_ld({pi, _Xml, _Attrs}, Acc) -> Acc; -json_ld({_, As, _} = Tag, Acc) -> +json_ld({<<"script">>, As, Es} = Tag, Acc) -> + case is_json_ld_script(As) of + true -> + lists:reverse(json_ld_script(Es), Acc); + false -> + json_ld_tag(Tag, Acc) + end; +json_ld({_, _As, _} = Tag, Acc) -> + json_ld_tag(Tag, Acc); +json_ld(_Text, Acc) -> + Acc. + +json_ld_tag({_, As, _} = Tag, Acc) -> case proplists:is_defined(<<"itemscope">>, As) of true -> {_Object, Acc1} = json_ld_item(Tag, Acc), Acc1; false -> json_ld_children(Tag, Acc) - end; -json_ld(_Text, Acc) -> - Acc. + end. json_ld_children({_Tag, _As, Es}, Acc) -> json_ld(Es, Acc). @@ -694,6 +704,36 @@ normalize_schema_itemtype(<<"https://schema.org/", Type/binary>>) -> normalize_schema_itemtype(Type) -> {type, Type}. +is_json_ld_script(As) -> + case proplists:get_value(<<"type">>, As) of + undefined -> + false; + Type -> + [Mime|_] = binary:split(z_string:to_lower(Type), <<";">>), + z_string:trim(Mime) =:= <<"application/ld+json">> + end. + +json_ld_script(Es) -> + case decode_json_ld(z_string:trim(fetch_text(Es, <<>>))) of + {ok, Map} when is_map(Map) -> + [Map]; + {ok, List} when is_list(List) -> + [ Map || Map <- List, is_map(Map) ]; + {error, _} -> + [] + end. + +decode_json_ld(<<>>) -> + {error, empty}; +decode_json_ld(Json) -> + try jsxrecord:decode(Json) of + JsonLD -> + {ok, JsonLD} + catch + Class:DecodeReason -> + {error, {Class, DecodeReason}} + end. + meta_link(_Name, undefined, _As, MD) -> MD; meta_link(_Name, <<>>, _As, MD) -> MD; meta_link(_Name, <<"undefined">>, _As, MD) -> MD; % Youtube... @@ -1101,6 +1141,50 @@ youtube_scoped_itemprop_html_meta_test() -> ?assertEqual(<<"Wes O'Donnell">>, maps:get(<<"name">>, Channel)), ok. +json_ld_script_html_meta_test() -> + Data = <<" + + ">>, + [{json_ld, [JsonLD]}] = html_meta(Data), + Graph = maps:get(<<"@graph">>, JsonLD), + ?assertEqual(<<"https://schema.org/">>, + maps:get(<<"schema">>, maps:get(<<"@context">>, JsonLD))), + ?assertEqual(2, length(Graph)), + ?assertEqual(<<"Untitled">>, + maps:get(<<"schema:name">>, hd(Graph))), + ?assertEqual(<<"Knowing together.\n\nYour data outlives your website.">>, + maps:get(<<"schema:description">>, lists:nth(2, Graph))), + ok. + +json_ld_script_metadata_property_test() -> + Data = <<" + + ">>, + {ok, MD} = fetch_data([], Data), + [One, Two] = p(json_ld, MD), + ?assertEqual(<<"One">>, maps:get(<<"name">>, One)), + ?assertEqual(<<"Two">>, maps:get(<<"name">>, Two)), + ok. + json_ld_by_id(ItemId, JsonLDs) -> hd([ JsonLD || JsonLD <- JsonLDs, maps:get(<<"@id">>, JsonLD, undefined) =:= ItemId ]). diff --git a/src/zotonic_stdlib.app.src b/src/zotonic_stdlib.app.src index d875c37..17613d0 100644 --- a/src/zotonic_stdlib.app.src +++ b/src/zotonic_stdlib.app.src @@ -5,7 +5,7 @@ {applications, [ kernel, stdlib, inets, tls_certificate_check, - cowlib, qdate_localtime + cowlib, jsxrecord, qdate_localtime ]}, {env, []}, {modules, []}, From f9d794e0a849b2e190c47a9271377deef9bcdc91 Mon Sep 17 00:00:00 2001 From: Marc Worrell Date: Tue, 30 Jun 2026 21:41:10 +0200 Subject: [PATCH 3/3] Add readme --- README.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/README.md b/README.md index 455ecdf..44aa20e 100644 --- a/README.md +++ b/README.md @@ -135,6 +135,27 @@ Fetch the first N bytes of a URL. Protection against too large return body. Fetch URL, extract metadata like mime type, title, description, images etc. +The returned metadata can be queried with `z_url_metadata:p/2`, for example: + + {ok, Metadata} = z_url_metadata:fetch(Url), + Title = z_url_metadata:p(title, Metadata), + Image = z_url_metadata:p(image, Metadata). + +JSON-LD data is available through the `json_ld` property: + + JsonLDs = z_url_metadata:p(json_ld, Metadata). + +This list contains decoded JSON-LD maps from: + + * `