<html xmlns:v="urn:schemas-microsoft-com:vml" xmlns:o="urn:schemas-microsoft-com:office:office" xmlns:w="urn:schemas-microsoft-com:office:word" xmlns:m="http://schemas.microsoft.com/office/2004/12/omml" xmlns="http://www.w3.org/TR/REC-html40">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
<meta name="Generator" content="Microsoft Word 15 (filtered medium)">
<!--[if !mso]><style>v\:* {behavior:url(#default#VML);}
o\:* {behavior:url(#default#VML);}
w\:* {behavior:url(#default#VML);}
.shape {behavior:url(#default#VML);}
</style><![endif]--><style><!--
/* Font Definitions */
@font-face
{font-family:Wingdings;
panose-1:5 0 0 0 0 0 0 0 0 0;}
@font-face
{font-family:"Cambria Math";
panose-1:2 4 5 3 5 4 6 3 2 4;}
@font-face
{font-family:Calibri;
panose-1:2 15 5 2 2 2 4 3 2 4;}
@font-face
{font-family:Consolas;
panose-1:2 11 6 9 2 2 4 3 2 4;}
/* Style Definitions */
p.MsoNormal, li.MsoNormal, div.MsoNormal
{margin:0cm;
margin-bottom:.0001pt;
font-size:11.0pt;
font-family:"Calibri",sans-serif;
mso-fareast-language:EN-US;}
a:link, span.MsoHyperlink
{mso-style-priority:99;
color:#0563C1;
text-decoration:underline;}
a:visited, span.MsoHyperlinkFollowed
{mso-style-priority:99;
color:#954F72;
text-decoration:underline;}
p
{mso-style-priority:99;
margin:0cm;
margin-bottom:.0001pt;
font-size:12.0pt;
font-family:"Times New Roman",serif;}
pre
{mso-style-priority:99;
mso-style-link:"HTML - förformaterad Char";
margin:0cm;
margin-bottom:.0001pt;
font-size:10.0pt;
font-family:"Courier New";}
p.MsoListParagraph, li.MsoListParagraph, div.MsoListParagraph
{mso-style-priority:34;
margin-top:0cm;
margin-right:0cm;
margin-bottom:0cm;
margin-left:36.0pt;
margin-bottom:.0001pt;
font-size:11.0pt;
font-family:"Calibri",sans-serif;
mso-fareast-language:EN-US;}
p.msonormal0, li.msonormal0, div.msonormal0
{mso-style-name:msonormal;
margin:0cm;
margin-bottom:.0001pt;
font-size:12.0pt;
font-family:"Times New Roman",serif;}
span.E-postmall19
{mso-style-type:personal;
font-family:"Calibri",sans-serif;
color:windowtext;}
span.E-postmall21
{mso-style-type:personal-reply;
font-family:"Calibri",sans-serif;
color:#1F497D;}
span.HTML-frformateradChar
{mso-style-name:"HTML - förformaterad Char";
mso-style-priority:99;
mso-style-link:"HTML - förformaterad";
font-family:"Courier New";}
.MsoChpDefault
{mso-style-type:export-only;
font-size:10.0pt;}
@page WordSection1
{size:612.0pt 792.0pt;
margin:70.85pt 70.85pt 70.85pt 70.85pt;}
div.WordSection1
{page:WordSection1;}
/* List Definitions */
@list l0
{mso-list-id:326174572;
mso-list-type:hybrid;
mso-list-template-ids:-1909434192 1005245440 69009411 69009413 69009409 69009411 69009413 69009409 69009411 69009413;}
@list l0:level1
{mso-level-start-at:6;
mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:Symbol;
mso-fareast-font-family:Calibri;
mso-bidi-font-family:"Times New Roman";}
@list l0:level2
{mso-level-number-format:bullet;
mso-level-text:o;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:"Courier New";}
@list l0:level3
{mso-level-number-format:bullet;
mso-level-text:\F0A7;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:Wingdings;}
@list l0:level4
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:Symbol;}
@list l0:level5
{mso-level-number-format:bullet;
mso-level-text:o;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:"Courier New";}
@list l0:level6
{mso-level-number-format:bullet;
mso-level-text:\F0A7;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:Wingdings;}
@list l0:level7
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:Symbol;}
@list l0:level8
{mso-level-number-format:bullet;
mso-level-text:o;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:"Courier New";}
@list l0:level9
{mso-level-number-format:bullet;
mso-level-text:\F0A7;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:Wingdings;}
@list l1
{mso-list-id:2087415144;
mso-list-template-ids:-12529080;}
@list l1:level1
{mso-level-number-format:bullet;
mso-level-text:\F0B7;
mso-level-tab-stop:36.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l1:level2
{mso-level-number-format:bullet;
mso-level-text:o;
mso-level-tab-stop:72.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;
mso-ansi-font-size:10.0pt;
font-family:"Courier New";
mso-bidi-font-family:"Times New Roman";}
@list l1:level3
{mso-level-number-format:bullet;
mso-level-text:\F0A7;
mso-level-tab-stop:108.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;
mso-ansi-font-size:10.0pt;
font-family:Wingdings;}
@list l1:level4
{mso-level-number-format:bullet;
mso-level-text:\F0A7;
mso-level-tab-stop:144.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;
mso-ansi-font-size:10.0pt;
font-family:Wingdings;}
@list l1:level5
{mso-level-number-format:bullet;
mso-level-text:\F0A7;
mso-level-tab-stop:180.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;
mso-ansi-font-size:10.0pt;
font-family:Wingdings;}
@list l1:level6
{mso-level-number-format:bullet;
mso-level-text:\F0A7;
mso-level-tab-stop:216.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;
mso-ansi-font-size:10.0pt;
font-family:Wingdings;}
@list l1:level7
{mso-level-number-format:bullet;
mso-level-text:\F0A7;
mso-level-tab-stop:252.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;
mso-ansi-font-size:10.0pt;
font-family:Wingdings;}
@list l1:level8
{mso-level-number-format:bullet;
mso-level-text:\F0A7;
mso-level-tab-stop:288.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;
mso-ansi-font-size:10.0pt;
font-family:Wingdings;}
@list l1:level9
{mso-level-number-format:bullet;
mso-level-text:\F0A7;
mso-level-tab-stop:324.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;
mso-ansi-font-size:10.0pt;
font-family:Wingdings;}
ol
{margin-bottom:0cm;}
ul
{margin-bottom:0cm;}
--></style><!--[if gte mso 9]><xml>
<o:shapedefaults v:ext="edit" spidmax="1026" />
</xml><![endif]--><!--[if gte mso 9]><xml>
<o:shapelayout v:ext="edit">
<o:idmap v:ext="edit" data="1" />
</o:shapelayout></xml><![endif]-->
</head>
<body lang="SV" link="#0563C1" vlink="#954F72">
<div class="WordSection1">
<p class="MsoNormal"><span lang="EN-GB" style="color:#1F497D">Ahh, interesting aspect! Seems to be that the dns lookups are stopped by quota but not other, later request, which gets -50 status.<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-GB" style="color:#1F497D"><o:p> </o:p></span></p>
<p class="MsoNormal"><span lang="EN-GB" style="color:#1F497D">Could it be that the “isn’t dns-lookedup” is discovered (giving -50) before the quota check?<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-GB" style="color:#1F497D"><o:p> </o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:2.4pt;margin-right:0cm;margin-bottom:2.4pt;margin-left:0cm;line-height:15.0pt;background:whitesmoke;word-break:break-all">
<span lang="EN-GB" style="font-size:10.0pt;font-family:Consolas;color:#333333;mso-fareast-language:SV">2019-06-14T10:36:29.804Z -50 - http://vasterasmotorstadion.se/xmlrpc.php?rsd RLLRE http://vasterasmotorstadion.se/ unknown #048 - - http://www.aktuellmotorsport.se
3t<o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:2.4pt;margin-right:0cm;margin-bottom:2.4pt;margin-left:0cm;line-height:15.0pt;background:whitesmoke;word-break:break-all">
<span lang="EN-GB" style="font-size:10.0pt;font-family:Consolas;color:#333333;mso-fareast-language:SV">2019-06-14T10:36:29.650Z -5003 - dns:vasterasmotorstadion.se RLLREP http://vasterasmotorstadion.se/xmlrpc.php?rsd unknown #048 - - http://www.aktuellmotorsport.se
Q:groupMaxAllKb<o:p></o:p></span></p>
<p class="MsoNormal" style="mso-margin-top-alt:2.4pt;margin-right:0cm;margin-bottom:2.4pt;margin-left:0cm;line-height:15.0pt;background:whitesmoke;word-break:break-all">
<span lang="EN-GB" style="font-size:10.0pt;font-family:Consolas;color:#333333;mso-fareast-language:SV">2019-06-14T10:36:29.427Z -5003 - dns:vasterasmotorstadion.se RLLREP http://vasterasmotorstadion.se/xmlrpc.php?rsd unknown #048 - - http://www.aktuellmotorsport.se
Q:groupMaxAllKb<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-GB" style="color:#1F497D"><o:p> </o:p></span></p>
<p class="MsoNormal"><span lang="EN-GB" style="color:#1F497D">How can I avoid this? Hmm, dns lookup shouldn’t be stopped by quota limit!<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-GB" style="color:#1F497D"><o:p> </o:p></span></p>
<p class="MsoNormal"><span lang="EN-GB" style="color:#1F497D">Hmm (2), if most of the -50 is when the domain is over quota limit, then I can ignore them!?<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-GB" style="font-size:9.0pt;font-family:"Arial",sans-serif;color:black;mso-fareast-language:SV">-----<br>
<br>
</span><span lang="EN-GB" style="font-family:"Arial",sans-serif;color:black;mso-fareast-language:SV">Peter Svanberg</span><span lang="EN-GB" style="color:#1F497D;mso-fareast-language:SV"><br>
</span><span lang="EN-GB" style="font-size:9.0pt;color:#1F497D;mso-fareast-language:SV"><br>
</span><span lang="EN-GB" style="font-family:"Arial",sans-serif;color:#1F497D;mso-fareast-language:SV">National Library of Sweden</span><span lang="EN-GB" style="color:#1F497D;mso-fareast-language:SV"><br>
</span><span lang="EN-GB" style="font-size:9.0pt;font-family:"Arial",sans-serif;color:#1F497D;mso-fareast-language:SV">Phone: +46 10 709
</span><span lang="EN-GB" style="font-size:9.0pt;font-family:"Arial",sans-serif;color:black;mso-fareast-language:SV">32 78</span><span lang="EN-GB" style="font-size:9.0pt;color:#1F497D;mso-fareast-language:SV"><br>
<br>
</span><span lang="EN-GB" style="font-size:9.0pt;font-family:"Arial",sans-serif;color:#1F497D;mso-fareast-language:SV">E-mail</span><span lang="EN-GB" style="font-size:9.0pt;color:#1F497D;mso-fareast-language:SV">:
</span><span lang="EN-GB" style="font-size:9.0pt;font-family:"Arial",sans-serif;color:black;mso-fareast-language:SV">peter.svanberg@kb.se</span><span lang="EN-GB" style="font-size:9.0pt;color:black;mso-fareast-language:SV"><br>
</span><span lang="EN-GB" style="font-size:9.0pt;font-family:"Arial",sans-serif;color:#1F497D;mso-fareast-language:SV">Web</span><span lang="EN-GB" style="font-size:9.0pt;color:#1F497D;mso-fareast-language:SV">:
</span><span lang="EN-GB" style="font-size:9.0pt;font-family:"Arial",sans-serif;color:#1F497D;mso-fareast-language:SV">www.kb.se</span><span lang="EN-GB" style="font-size:9.0pt;color:#1F497D;mso-fareast-language:SV"><br>
<br>
</span><span lang="EN-GB" style="color:#1F497D;mso-fareast-language:SV"><o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-GB" style="color:#1F497D"><o:p> </o:p></span></p>
<p class="MsoNormal"><span lang="EN-GB" style="color:#1F497D"><o:p> </o:p></span></p>
<div>
<div style="border:none;border-top:solid #E1E1E1 1.0pt;padding:3.0pt 0cm 0cm 0cm">
<p class="MsoNormal"><b><span lang="EN-GB" style="mso-fareast-language:SV">Från:</span></b><span lang="EN-GB" style="mso-fareast-language:SV"> NetarchiveSuite-users <netarchivesuite-users-bounces@ml.sbforge.org>
<b>För </b>Colin Samuel Rosenthal<br>
<b>Skickat:</b> den 14 juni 2019 14:28<br>
<b>Till:</b> netarchivesuite-users@ml.sbforge.org<br>
<b>Ämne:</b> Re: [Netarchivesuite-users] Lots of -50 status codes<o:p></o:p></span></p>
</div>
</div>
<p class="MsoNormal"><span lang="EN-GB"><o:p> </o:p></span></p>
<div id="divtagdefaultwrapper">
<p><span style="font-family:"Calibri",sans-serif;color:black">I know we got rid of a lot -50 codes once we fixed our queue assignment policy to always queue dns lookups in the same queue as urls for the corresponding domain. But all that should be fixed in
5.5 . Do you see any problems with dns lookups?<o:p></o:p></span></p>
<p><span style="font-family:"Calibri",sans-serif;color:black"><o:p> </o:p></span></p>
<p><span style="font-family:"Calibri",sans-serif;color:black">cheers,<o:p></o:p></span></p>
<p><span style="font-family:"Calibri",sans-serif;color:black">Colin<o:p></o:p></span></p>
<p><span style="font-family:"Calibri",sans-serif;color:black"><o:p> </o:p></span></p>
<div id="Signature">
<div name="divtagdefaultwrapper">
<div>
<p class="MsoNormal"><span style="font-size:12.0pt;color:black;mso-fareast-language:SV">--<o:p></o:p></span></p>
</div>
<div>
<p class="MsoNormal"><span style="font-size:12.0pt;color:black;mso-fareast-language:SV">Colin Rosenthal PhD<o:p></o:p></span></p>
</div>
<div>
<p class="MsoNormal"><span style="font-size:12.0pt;color:black;mso-fareast-language:SV">Senior IT Consultant<o:p></o:p></span></p>
</div>
<div>
<p class="MsoNormal"><span style="font-size:12.0pt;color:black;mso-fareast-language:SV">Royal Danish Library (Aarhus)<o:p></o:p></span></p>
</div>
</div>
</div>
</div>
<div class="MsoNormal" align="center" style="text-align:center"><span style="font-size:12.0pt;font-family:"Times New Roman",serif;mso-fareast-language:SV">
<hr size="2" width="98%" align="center">
</span></div>
<div id="divRplyFwdMsg">
<p class="MsoNormal"><b><span style="color:black;mso-fareast-language:SV">From:</span></b><span style="color:black;mso-fareast-language:SV"> NetarchiveSuite-users <<a href="mailto:netarchivesuite-users-bounces@ml.sbforge.org">netarchivesuite-users-bounces@ml.sbforge.org</a>>
on behalf of Peter Svanberg <<a href="mailto:Peter.Svanberg@kb.se">Peter.Svanberg@kb.se</a>><br>
<b>Sent:</b> Friday, June 14, 2019 11:56:42 AM<br>
<b>To:</b> <a href="mailto:netarchivesuite-users@ml.sbforge.org">netarchivesuite-users@ml.sbforge.org</a><br>
<b>Subject:</b> [Netarchivesuite-users] Lots of -50 status codes</span><span style="font-size:12.0pt;font-family:"Times New Roman",serif;mso-fareast-language:SV">
<o:p></o:p></span></p>
<div>
<p class="MsoNormal"><span style="font-size:12.0pt;font-family:"Times New Roman",serif;mso-fareast-language:SV"> <o:p></o:p></span></p>
</div>
</div>
<div>
<p class="MsoNormal">Hello all!<o:p></o:p></p>
<p class="MsoNormal"><o:p> </o:p></p>
<p class="MsoNormal"><span lang="EN-GB">I get quite a lot of -50 status lines in my crawl.log (test snapshot runs, NetarchiveSuite 5.5). I haven’t seen any pattern in which kind of
<a href="URL:s">URL:s</a> that gets this – sometimes it is <a href="URL:s">URL:s</a> which redirects to entrance/top level page (should be banned!), sometimes it’s quite ordinary
<a href="URL:s">URL:s</a>, often to images, I think. And manual fetching later always works.<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-GB"><o:p> </o:p></span></p>
<p class="MsoNormal"><span lang="EN-GB">One pattern is that it seems as if it either<o:p></o:p></span></p>
<p class="MsoListParagraph" style="text-indent:-18.0pt;mso-list:l0 level1 lfo2"><![if !supportLists]><span lang="EN-GB" style="font-family:Symbol"><span style="mso-list:Ignore">·<span style="font:7.0pt "Times New Roman"">
</span></span></span><![endif]><span lang="EN-GB">all requests to a certain host are -50, or<o:p></o:p></span></p>
<p class="MsoListParagraph" style="text-indent:-18.0pt;mso-list:l0 level1 lfo2"><![if !supportLists]><span lang="EN-GB" style="font-family:Symbol"><span style="mso-list:Ignore">·<span style="font:7.0pt "Times New Roman"">
</span></span></span><![endif]><span lang="EN-GB">first a series of 200 and then ditto -50<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-GB">so they are not intermixed. That could imply some problem – or automatic blocking after a while? – on the host.<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-GB"><o:p> </o:p></span></p>
<p class="MsoNormal"><span lang="EN-GB">What are your experiences?<o:p></o:p></span></p>
<p class="MsoNormal"><o:p> </o:p></p>
<p class="MsoNormal" style="margin-bottom:12.0pt"><span lang="EN-GB" style="font-size:9.0pt;font-family:"Arial",sans-serif;color:black;mso-fareast-language:SV">-----<br>
<br>
</span><span lang="EN-GB" style="font-family:"Arial",sans-serif;color:black;mso-fareast-language:SV">Peter Svanberg</span><span lang="EN-GB" style="mso-fareast-language:SV"><br>
</span><span lang="EN-GB" style="font-size:9.0pt;mso-fareast-language:SV"><br>
</span><span lang="EN-GB" style="font-family:"Arial",sans-serif;mso-fareast-language:SV">National Library of Sweden</span><span lang="EN-GB" style="mso-fareast-language:SV"><br>
</span><span lang="EN-GB" style="font-size:9.0pt;font-family:"Arial",sans-serif;mso-fareast-language:SV">Phone: +46 10 709
<span style="color:black">32 78</span></span><span lang="EN-GB" style="font-size:9.0pt;mso-fareast-language:SV"><br>
<br>
</span><span lang="EN-GB" style="font-size:9.0pt;font-family:"Arial",sans-serif;mso-fareast-language:SV">E-mail</span><span lang="EN-GB" style="font-size:9.0pt;mso-fareast-language:SV">:
</span><span lang="EN-GB" style="font-size:9.0pt;font-family:"Arial",sans-serif;color:black;mso-fareast-language:SV"><a href="mailto:peter.svanberg@kb.se">peter.svanberg@kb.se</a></span><span lang="EN-GB" style="font-size:9.0pt;color:black;mso-fareast-language:SV"><br>
</span><span lang="EN-GB" style="font-size:9.0pt;font-family:"Arial",sans-serif;mso-fareast-language:SV">Web</span><span lang="EN-GB" style="font-size:9.0pt;mso-fareast-language:SV">:
</span><span lang="EN-GB" style="font-size:9.0pt;font-family:"Arial",sans-serif;mso-fareast-language:SV"><a href="http://www.kb.se">www.kb.se</a></span><span lang="EN-GB" style="mso-fareast-language:SV"><o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-GB"><o:p> </o:p></span></p>
<p class="MsoNormal"><span lang="EN-GB"><o:p> </o:p></span></p>
</div>
</div>
</body>
</html>