Repository navigation
Expand file tree
/
Copy pathcrawl.conf
More file actions
73 lines (60 loc) · 2.51 KB
/
Copy pathcrawl.conf
File metadata and controls
73 lines (60 loc) · 2.51 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
# This file is managed by Puppet. DO NOT EDIT.
# modules/global/templates/res/crawl.conf.erb
[crawler]
detach=yes
verbose=no
threads=1
[cluster]
name=anansi
registry=http://localhost:2379/
environment=live
[processor]
name=lod
[lod:licenses]
predicate="http://purl.org/dc/terms/rights"
predicate="http://purl.org/dc/terms/license"
predicate="http://purl.org/dc/terms/accessRights"
predicate="http://creativecommons.org/ns#license"
predicate="http://www.w3.org/1999/xhtml/vocab#license"
predicate="http://www.w3.org/1999/xhtml/vocab#copyright"
whitelist="http://creativecommons.org/publicdomain/zero/1.0/"
whitelist="http://id.loc.gov/about/"
whitelist="http://creativecommons.org/licenses/by/4.0/"
whitelist="http://creativecommons.org/licenses/by-sa/4.0/"
whitelist="http://reference.data.gov.uk/id/open-government-licence"
whitelist="http://bbcarchdev.github.io/licences/dps/1.0#id"
whitelist="http://creativecommons.org/licenses/by/1.0/"
whitelist="http://creativecommons.org/licenses/by-sa/1.0/"
whitelist="http://creativecommons.org/licenses/by/2.5/"
whitelist="http://creativecommons.org/licenses/by-sa/2.5/"
whitelist="http://creativecommons.org/licenses/by/3.0/"
whitelist="http://creativecommons.org/licenses/by-sa/3.0/"
whitelist="http://creativecommons.org/licenses/by/3.0/us/"
whitelist="http://creativecommons.org/licenses/by-sa/3.0/us/"
whitelist="https://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/"
whitelist="http://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/"
whitelist="https://www.nationalarchives.gov.uk/doc/open-government-licence/version/2/"
whitelist="http://www.nationalarchives.gov.uk/doc/open-government-licence/version/2/"
whitelist="https://www.nationalarchives.gov.uk/doc/open-government-licence/version/1/"
whitelist="http://www.nationalarchives.gov.uk/doc/open-government-licence/version/1/"
whitelist="http://www.ordnancesurvey.co.uk/business-and-government/licensing/using-creating-data-with-os-products/os-opendata.html"
whitelist="http://www.ordnancesurvey.co.uk/oswebsite/opendata/licence/docs/licence.pdf"
whitelist="http://opendatacommons.org/licenses/pddl/1.0/"
whitelist="http://opendatacommons.org/licenses/by/1.0/"
whitelist="http://opendatacommons.org/licenses/odbl/1.0/"
whitelist="http://collection.britishmuseum.org/licensing.html"
[cache]
uri=s3://anansi/
endpoint=s3
username=x
password=x
[queue]
name=db
uri=pgsql://postgres:postgres@postgres/anansi
debug-queries=no
debug-errors=yes
[policy:schemes]
whitelist=http,https,ftp
[log]
level=debug
facility=daemon