| #Proxy Configuration, if not specified: use no proxy or use the JRE proxy config via system properties, if given. |
| #proxyHost=localhost |
| #proxyPort=80 |
| |
| # HTTP Socket timeout for the web crawler service. |
| # If a sync request blocks for longer than the configured timeout (in ms), the request will be interrupted |
| # Default value: none (infinite) |
| socketTimeout=300000 |
| |
| # String to use as "User-Agent" header. |
| userAgent=SMILA (http://wiki.eclipse.org/SMILA/UserAgent; smila-dev@eclipse.org) |
| |
| # Crawling behavior on 40x-HTTP-Response of robots.txt |
| allowAllOn40xForRobots=false |
