开通博客赚积分
发布资源赚积分
分类

源码开发语言/平台
当前位置：首页 > 源码/资料 > Internet/网络编程 > 浏览器 > 查看源码
wwwoffles.c
资源名称：wwwoffle-2.5e.tgz　[点击查看]
上传用户：seven77cht
上传日期：2007-01-04
资源大小：486k
文件大小：48k
源码类别：

浏览器

开发平台：

Unix_Linux

wwwoffles.c：源码内容
  1. /***************************************
  2.   $Header: /home/amb/wwwoffle/RCS/wwwoffles.c 2.141 1999/12/04 11:38:21 amb Exp $
  4.   WWWOFFLE - World Wide Web Offline Explorer - Version 2.5c.
  5.   A server to fetch the required pages.
  6.   ******************/ /******************
  7.   Written by Andrew M. Bishop
  9.   This file Copyright 1996,97,98,99 Andrew M. Bishop
  10.   It may be distributed under the GNU Public License, version 2, or
  11.   any higher version.  See section COPYING of the GNU Public license
  12.   for conditions under which this file may be redistributed.
  13.   ***************************************/
  16. #include <stdio.h>
  17. #include <stdlib.h>
  18. #include <string.h>
  20. #include <sys/types.h>
  21. #include <sys/stat.h>
  22. #include <time.h>
  23. #include <signal.h>
  24. #include <unistd.h>
  25. #include <fcntl.h>
  27. #include "wwwoffle.h"
  28. #include "document.h"
  29. #include "misc.h"
  30. #include "proto.h"
  31. #include "config.h"
  32. #include "sockets.h"
  33. #include "errors.h"
  36. static int ssl_tunnel(int client,URL *Url,Header *request_head);
  37. static void uninstall_sighandlers(void);
  40. /*+ The mode of operation of the server. +*/
  41. typedef enum _Mode
  42. {
  43.  None,                          /*+ Undecided. +*/
  45.  Real,                          /*+ From server host to cache and client. +*/
  46.  RealNoCache,                   /*+ From server host to client. +*/
  47.  RealRefresh,                   /*+ Refresh the page, forced from index. +*/
  48.  RealNoPassword,                /*+ From server host to cache, not using supplied password. +*/
  49.  SpoolOrReal,                   /*+ Spool if already cached else Real. +*/
  51.  Fetch,                         /*+ From server host to cache. +*/
  52.  FetchNoPassword,               /*+ From server host to cache, not using supplied password. +*/
  54.  Spool,                         /*+ From cache to client. +*/
  55.  SpoolGet,                      /*+ Not in cache so record request in outgoing. +*/
  56.  SpoolWillGet,                  /*+ Not in cache but is in outgoing. +*/
  57.  SpoolRefresh,                  /*+ Refresh the page, forced from refresh page. +*/
  58.  SpoolPragma,                   /*+ Refresh the page, forced from browser by 'Pragma: no-cache'. +*/
  60.  SpoolInternal                  /*+ The page is one that has been internally generated by WWWOFFLE. +*/
  61. }
  62. Mode;
  65. /*++++++++++++++++++++++++++++++++++++++
  66.   The main server program.
  68.   int wwwoffles Returns the exit status.
  70.   int online Whether the demon is online or not.
  72.   int browser Set to true if there is a browser.
  74.   int client The file descriptor of the client.
  75.   ++++++++++++++++++++++++++++++++++++++*/
  77. int wwwoffles(int online,int browser,int client)
  78. {
  79.  int outgoing=-1,spool=-1,tmpclient=-1;
  80.  int fetch_again=0;
  81.  char *proxy_auth,*proxy_user;
  82.  char *aliasProto=NULL,*aliasHost=NULL,*aliasPath=NULL;
  83.  Header *request_head=NULL,*reply_head=NULL;
  84.  Body *request_body=NULL,*reply_body=NULL;
  85.  int reply_status=-1,head_only=0;
  86.  char *url;
  87.  URL *Url=NULL,*Urlpw=NULL;
  88.  Mode mode=None;
  89.  int outgoing_exists=0,lasttime_exists=0;
  90.  time_t spool_exists=0,spool_exists_pw=0;
  91.  int offline_request=1;
  92.  char *is_client_wwwoffle=NULL,*is_client_htdig=NULL;
  94.  /* Initialise things. */
  96.  uninstall_sighandlers();
  98.  InitErrorHandler("wwwoffles",-1,-1); /* change name nothing else */
  100.  if(online==1 && browser)
  101.     mode=Real;
  102.  else if(online==1 && !browser)
  103.     mode=Fetch;
  104.  else if(online==-1 && browser)
  105.     mode=SpoolOrReal;
  106.  else if(!online && browser)
  107.     mode=Spool;
  108.  else
  109.     PrintMessage(Fatal,"Started in a mode that is not allowed (online=%d, browser=%d).",online,browser);
  111.  /* mode = Spool, Real, SpoolOrReal or Fetch */
  113.  /* Set up the client file. */
  115.  if(client==-1 && mode!=Fetch)
  116.     PrintMessage(Fatal,"Cannot use client file descriptor %d.",client);
  118.  if(mode!=Fetch)
  119.    {
  120.     tmpclient=CreateTempSpoolFile();
  122.     if(tmpclient==-1)
  123.       {
  124.        PrintMessage(Warning,"Cannot open temporary spool file [%!s].");
  125.        HTMLMessage(client,500,"WWWOFFLE Server Error",NULL,"ServerError",
  126.                    "error","Cannot open temporary file.",
  127.                    NULL);
  128.        exit(1);
  129.       }
  130.    }
  132.  /* Set up the outgoing file. */
  134.  if(mode==Fetch)
  135.    {
  136.     outgoing=OpenOutgoingSpoolFile(1);
  137.     init_buffer(outgoing);
  139.     if(outgoing==-1)
  140.       {PrintMessage(Inform,"No more outgoing requests.");exit(3);}
  141.    }
  143.  /* Get the URL from the request. */
  145.  if(mode==Real || mode==Spool || mode==SpoolOrReal)
  146.     url=ParseRequest(client,&request_head,&request_body);
  147.  else /* mode==Fetch */
  148.     url=ParseRequest(outgoing,&request_head,&request_body);
  150.  if(DebugLevel==ExtraDebug)
  151.    {
  152.     if(request_head)
  153.        PrintMessage(ExtraDebug,"Incoming Request Head (from browser)n%s",HeaderString(request_head));
  154.     else
  155.        PrintMessage(ExtraDebug,"Incoming Request Head (from browser) is empty");
  157.     if(request_body)
  158.        if(strcmp(request_head->method,"POST")) /* only POST is guaranteed to be ASCII. */
  159.           PrintMessage(ExtraDebug,"Incoming Request Body (from browser) is %d bytes of binary datan",request_body->length);
  160.        else
  161.           PrintMessage(ExtraDebug,"Incoming Request Body (from browser)n%s",request_body->content);
  162.    }
  164.  if(!url)
  165.    {
  166.     PrintMessage(Warning,"Could not parse HTTP request (%s).",request_head?"Parse error":"Empty request");
  167.     if(mode!=Fetch)
  168.        HTMLMessage(client,500,"WWWOFFLE Server Error",NULL,"ServerError",
  169.                    "error",request_head?"Cannot parse the HTTP request":"The HTTP request was empty",
  170.                    NULL);
  171.     exit(1);
  172.    }
  174.  Url=SplitURL(url);
  176.  /* Authenticate the user with the proxy */
  178.  proxy_auth=GetHeader(request_head,"Proxy-Authorization",NULL);
  179.  proxy_user=IsAllowedConnectUser(proxy_auth);
  181.  if(!proxy_user)
  182.    {
  183.     PrintMessage(Inform,"HTTP Proxy connection rejected from unauthenticated user."); /* Used in audit-usage.pl */
  184.     HTMLMessageHead(tmpclient,407,"WWWOFFLE Proxy Authentication Required",
  185.                     "Proxy-Authenticate","Basic realm="wwwoffle-proxy"",
  186.                     NULL);
  187.     HTMLMessageBody(tmpclient,"ProxyAuthFail",
  188.                     NULL);
  189.     mode=SpoolInternal; goto spoolinternal;
  190.    }
  191.  else if(proxy_auth)
  192.     PrintMessage(Inform,"HTTP Proxy connection from user '%s'.",proxy_user); /* Used in audit-usage.pl */
  194.  /* Check the HTTP method that is requested */
  196.  if(!strcasecmp(request_head->method,"CONNECT"))
  197.    {
  198.     PrintMessage(Inform,"SSL='%s'.",Url->host); /* Used in audit-usage.pl */
  200.     if(mode==Real || mode==SpoolOrReal || (mode==Spool && IsLocalNetHost(Url->host)))
  201.       {
  202.        if(IsSSLAllowedPort(Url->host))
  203.          {
  204.           int ret=ssl_tunnel(client,Url,request_head);
  205.           exit(ret);
  206.          }
  207.        else
  208.          {
  209.           PrintMessage(Warning,"A SSL proxy connection for %s was received but is not allowed.",Url->host);
  210.           HTMLMessage(tmpclient,500,"WWWOFFLE Server Error",NULL,"ServerError",
  211.                       "error","SSL proxy connection to specified port is not allowed.",
  212.                       NULL);
  213.           mode=SpoolInternal; goto spoolinternal;
  214.          }
  215.       }
  216.     else /* mode==Fetch || mode==Spool */
  217.       {
  218.        PrintMessage(Warning,"A SSL proxy connection for %s was received but wwwoffles is in wrong mode.",Url->host);
  219.        if(mode==Fetch)
  220.          {
  221.           if(client!=-1)
  222.              write_formatted(client,"Cannot fetch %s [HTTP method 'CONNECT' not supported in this mode]n",Url->name);
  223.           exit(1);
  224.          }
  225.        else
  226.          {
  227.           HTMLMessage(tmpclient,500,"WWWOFFLE Server Error",NULL,"ServerError",
  228.                       "error","SSL proxy connection while offline is not allowed.",
  229.                       NULL);
  230.           mode=SpoolInternal; goto spoolinternal;
  231.          }
  232.       }
  233.    }
  234.  else if(strcmp(request_head->method,"GET") &&
  235.          strcmp(request_head->method,"HEAD") &&
  236.          strcmp(request_head->method,"POST") &&
  237.          strcmp(request_head->method,"PUT"))
  238.    {
  239.     PrintMessage(Warning,"The requested method '%s' is not supported.",request_head->method);
  240.     if(mode==Fetch)
  241.       {
  242.        if(client!=-1)
  243.           write_formatted(client,"Cannot fetch %s [The %s method is not supported]n",Url->name,request_head->method);
  244.        exit(1);
  245.       }
  246.     else
  247.       {
  248.        HTMLMessage(tmpclient,501,"WWWOFFLE Method Unsupported",NULL,"MethodUnsupported",
  249.                    "method",request_head->method,
  250.                    "protocol","all",
  251.                    NULL);
  252.        mode=SpoolInternal; goto spoolinternal;
  253.       }
  254.    }
  255.  else if(!strcmp(request_head->method,"HEAD"))
  256.    {
  257.     strcpy(request_head->method,"GET");
  258.     request_head->size-=1;
  259.     head_only=1;
  260.    }
  262.  PrintMessage(Inform,"URL='%s'%s.",Url->name,Url->user?" (With username/password)":""); /* Used in audit-usage.pl */
  263.  PrintMessage(Debug,"proto='%s'; host='%s'; path='%s'; args='%s'; user:pass='%s:%s'.",
  264.               Url->proto,Url->host,Url->path,Url->args,Url->user,Url->pass);
  266.  /* Check for an alias. */
  268.  if(IsAliased(Url->proto,Url->host,Url->path,&aliasProto,&aliasHost,&aliasPath))
  269.    {
  270.     URL *newUrl;
  271.     char *newurl=(char*)malloc(strlen(aliasProto)+strlen(aliasHost)+strlen(aliasPath)-
  272.                                strlen(Url->path)+strlen(Url->pathp)+8);
  274.     sprintf(newurl,"%s://%s%s%s",aliasProto,aliasHost,aliasPath,Url->pathp+strlen(Url->path));
  276.     newUrl=SplitURL(newurl);
  278.     if(Url->user)
  279.        AddURLPassword(newUrl,Url->user,Url->pass);
  281.     url=newurl;
  282.     FreeURL(Url);
  283.     Url=newUrl;
  285.     PrintMessage(Inform,"Aliased URL='%s'%s.",newUrl->name,newUrl->user?" (With username/password)":"");
  286.     PrintMessage(Debug,"Aliased proto='%s'; host='%s'; path='%s'; args='%s'; user:pass='%s:%s'.",
  287.                  newUrl->proto,newUrl->host,newUrl->path,newUrl->args,newUrl->user,newUrl->pass);
  288.    }
  290.  /* Initial sanity checks. */
  292.  if(IsNotGot(Url->proto,Url->host,Url->path,Url->args))
  293.    {
  294.     char *replace = NotGotReplacement(Url->proto,Url->host,Url->path,Url->args);
  296.     if(!replace)
  297.       {
  298.        PrintMessage(Inform,"The server '%s://%s' and/or path '%s' is on the list not to get.",Url->proto,Url->host,Url->path);
  299.        if(mode==Fetch)
  300.           exit(0);
  301.        else
  302.          {
  303.           HTMLMessage(tmpclient,404,"WWWOFFLE Host Not Got",NULL,"HostNotGot",
  304.                       "url",Url->name,
  305.                       NULL);
  306.           mode=SpoolInternal; goto spoolinternal;
  307.          }
  308.       }
  309.     else
  310.       {
  311.        char *newurl=(char*)malloc(strlen(replace)+1);
  312.        URL *newUrl=SplitURL(replace);
  314.        strcpy(newurl,replace);
  316.        url=newurl;
  317.        FreeURL(Url);
  318.        Url=newUrl;
  320.        PrintMessage(Inform,"Replaced URL='%s'%s.",newUrl->name,newUrl->user?" (With username/password)":"");
  321.        PrintMessage(Debug,"Replaced proto='%s'; host='%s'; path='%s'; args='%s'; user:pass='%s:%s'.",
  322.                     newUrl->proto,newUrl->host,newUrl->path,newUrl->args,newUrl->user,newUrl->pass);
  323.       }
  324.    }
  326.  if(!Url->Protocol)
  327.    {
  328.     PrintMessage(Inform,"The protocol '%s' is not available.",Url->proto);
  329.     if(mode==Fetch)
  330.       {
  331.        if(client!=-1)
  332.           write_formatted(client,"Cannot fetch %s [Protocol not available]n",Url->name);
  333.        exit(1);
  334.       }
  335.     else
  336.       {
  337.        HTMLMessage(tmpclient,404,"WWWOFFLE Illegal Protocol",NULL,"IllegalProtocol",
  338.                    "url",Url->name,
  339.                    "protocol",Url->proto,
  340.                    NULL);
  341.        mode=SpoolInternal; goto spoolinternal;
  342.       }
  343.    }
  345.  if((!strcmp(request_head->method,"POST") && !Url->Protocol->postable) ||
  346.     (!strcmp(request_head->method,"PUT") && !Url->Protocol->putable))
  347.    {
  348.     PrintMessage(Warning,"The requested method '%s' is not supported for the %s protocol.",request_head->method,Url->Protocol->name);
  349.     if(mode==Fetch)
  350.       {
  351.        if(client!=-1)
  352.           write_formatted(client,"Cannot fetch %s [The %s method is not supported for %s]n",Url->name,request_head->method,Url->Protocol->name);
  353.        exit(1);
  354.       }
  355.     else
  356.       {
  357.        HTMLMessage(tmpclient,501,"WWWOFFLE Method Unsupported",NULL,"MethodUnsupported",
  358.                    "method",request_head->method,
  359.                    "protocol",Url->Protocol->name,
  360.                    NULL);
  361.        mode=SpoolInternal; goto spoolinternal;
  362.       }
  363.    }
  365.  /* Change the mode based on the URL as required. */
  367.  /* mode = Spool, Real, SpoolOrReal or Fetch */
  369.  /* A local URL. */
  371.  if(Url->local)
  372.    {
  373.     if(!strncmp("/refresh",Url->path,8))
  374.       {
  375.        int recurse=0;
  377.        char *newurl=RefreshPage(tmpclient,Url,request_body,&recurse);
  379.        if(!newurl)
  380.          {mode=SpoolInternal; goto spoolinternal;}
  381.        else
  382.          {
  383.           URL *newUrl=SplitURL(newurl);
  385.           if(Url->user)
  386.              AddURLPassword(newUrl,Url->user,Url->pass);
  388.           if(newUrl->args && *newUrl->args=='!')
  389.             {
  390.              PrintMessage(Inform,"It is not possible to refresh a URL that used the POST/PUT method.");
  391.              if(mode==Fetch)
  392.                {
  393.                 if(client!=-1)
  394.                    write_formatted(client,"Cannot fetch %s [Reply from a POST/PUT method]n",Url->name);
  395.                 exit(1);
  396.                }
  397.              else
  398.                {
  399.                 HTMLMessage(tmpclient,404,"WWWOFFLE Cant Refresh POST/PUT",NULL,"CantRefreshPosted",
  400.                             "url",newUrl->name,
  401.                             NULL);
  402.                 mode=SpoolInternal; goto spoolinternal;
  403.                }
  404.             }
  405.           else if(recurse)
  406.             {
  407.              fetch_again=1;
  409.              if(mode!=Fetch)
  410.                {
  411.                 mode=SpoolGet;
  413.                 if(!Url->args || *Url->args!='!')
  414.                   {
  415.                    free(newurl);
  416.                    FreeURL(newUrl);
  417.                    newurl=(char*)malloc(strlen(url)+1);
  418.                    strcpy(newurl,url);
  419.                    newUrl=SplitURL(newurl);
  420.                    if(Url->user)
  421.                       AddURLPassword(newUrl,Url->user,Url->pass);
  422.                   }
  424.                 FreeHeader(request_head);
  425.                 request_head=RequestURL(newUrl,NULL);
  426.                 if(request_body)
  427.                    FreeBody(request_body);
  428.                 request_body=NULL;
  429.                }
  430.             }
  431.           else
  432.             {
  433.              if(mode==Spool)
  434.                 mode=SpoolRefresh;
  435.              else
  436.                {
  437.                 DeleteWebpageSpoolFile(newUrl,0);
  438.                 mode=RealRefresh;
  439.                }
  440.             }
  442.           url=newurl;
  443.           FreeURL(Url);
  444.           Url=newUrl;
  446.           PrintMessage(Inform,"Refresh URL='%s'%s.",newUrl->name,newUrl->user?" (With username/password)":"");
  447.           PrintMessage(Debug,"Refresh proto='%s'; host='%s'; path='%s'; args='%s'; user:pass='%s:%s'.",
  448.                        newUrl->proto,newUrl->host,newUrl->path,newUrl->args,newUrl->user,newUrl->pass);
  449.          }
  450.       }
  451.     else if(mode==Fetch)
  452.       {
  453.        PrintMessage(Inform,"The request to fetch a page from the local host is ignored.");
  454.        if(client!=-1)
  455.           write_formatted(client,"Cannot fetch %s [On local host]n",Url->name);
  456.        exit(1);
  457.       }
  458.     else if(!strncmp("/index/",Url->path,7))
  459.       {
  460.        IndexPage(tmpclient,Url);
  461.        mode=SpoolInternal; goto spoolinternal;
  462.       }
  463.     else if(!strncmp("/control/",Url->path,9))
  464.       {
  465.        ControlPage(tmpclient,Url,request_head,request_body);
  466.        mode=SpoolInternal; goto spoolinternal;
  467.       }
  468.     else if(!strncmp("/monitor",Url->path,8))
  469.       {
  470.        MonitorPage(tmpclient,Url,request_body);
  471.        mode=SpoolInternal; goto spoolinternal;
  472.       }
  473.     else if((!strchr(Url->path+1,'/') || !strncmp("/local/",Url->path,7)) && !Url->args)
  474.       {
  475.        LocalPage(tmpclient,Url->path,request_head);
  476.        mode=SpoolInternal; goto spoolinternal;
  477.       }
  478.     else if(!strncmp("/htdig/",Url->path,7))
  479.       {
  480.        HTDigPage(tmpclient,Url,request_head);
  481.        mode=SpoolInternal; goto spoolinternal;
  482.       }
  483.     else
  484.       {
  485.        int i;
  487.        for(i=0;i<NProtocols;i++)
  488.           if(!strncmp(Protocols[i].name,Url->pathp+1,strlen(Protocols[i].name)) &&
  489.              Url->pathp[strlen(Protocols[i].name)+1]=='/')
  490.             {
  491.              url=(char*)malloc(strlen(Url->pathp)+4);
  492.              Url->pathp[strlen(Protocols[i].name)+1]=0;
  493.              sprintf(url,"%s://%s",Url->pathp+1,&Url->pathp[strlen(Protocols[i].name)+2]);
  494.              Url->pathp[strlen(Protocols[i].name)+1]='/';
  495.              break;
  496.             }
  498.        if(i==NProtocols)
  499.          {
  500.           PrintMessage(Inform,"The requested URL '%s' does not exist on the local server.",Url->pathp);
  501.           HTMLMessage(tmpclient,404,"WWWOFFLE Page Not Found",NULL,"PageNotFound",
  502.                       "url",Url->name,
  503.                       NULL);
  504.           mode=SpoolInternal; goto spoolinternal;
  505.          }
  506.        else
  507.          {
  508.           FreeURL(Url);
  509.           Url=SplitURL(url);
  510.          }
  511.       }
  513.     /* Repeat initial sanity checks. */
  515.     if(IsNotGot(Url->proto,Url->host,Url->path,Url->args))
  516.       {
  517.        char *replace = NotGotReplacement(Url->proto,Url->host,Url->path,Url->args);
  519.        if(!replace)
  520.          {
  521.           PrintMessage(Inform,"The server '%s://%s' and/or path '%s' is on the list not to get.",Url->proto,Url->host,Url->path);
  522.           if(mode==Fetch)
  523.              exit(0);
  524.           else
  525.             {
  526.              HTMLMessage(tmpclient,404,"WWWOFFLE Host Not Got",NULL,"HostNotGot",
  527.                          "url",Url->name,
  528.                          NULL);
  529.              mode=SpoolInternal; goto spoolinternal;
  530.             }
  531.          }
  532.        else
  533.          {
  534.           char *newurl=(char*)malloc(strlen(replace)+1);
  535.           URL *newUrl=SplitURL(replace);
  537.           strcpy(newurl,replace);
  539.           url=newurl;
  540.           FreeURL(Url);
  541.           Url=newUrl;
  543.           PrintMessage(Inform,"Replaced URL='%s'%s.",newUrl->name,newUrl->user?" (With username/password)":"");
  544.           PrintMessage(Debug,"Replaced proto='%s'; host='%s'; path='%s'; args='%s'; user:pass='%s:%s'.",
  545.                        newUrl->proto,newUrl->host,newUrl->path,newUrl->args,newUrl->user,newUrl->pass);
  546.          }
  547.       }
  549.     if(!Url->Protocol)
  550.       {
  551.        PrintMessage(Inform,"The protocol '%s' is not available.",Url->proto);
  552.        if(mode==Fetch)
  553.          {
  554.           if(client!=-1)
  555.              write_formatted(client,"Cannot fetch %s [Protocol not available]n",Url->name);
  556.           exit(1);
  557.          }
  558.        else
  559.          {
  560.           HTMLMessage(tmpclient,404,"WWWOFFLE Illegal Protocol",NULL,"IllegalProtocol",
  561.                       "url",Url->name,
  562.                       "protocol",Url->proto,
  563.                       NULL);
  564.           mode=SpoolInternal; goto spoolinternal;
  565.          }
  566.       }
  568.     if((!strcmp(request_head->method,"POST") && !Url->Protocol->postable) ||
  569.        (!strcmp(request_head->method,"PUT") && !Url->Protocol->putable))
  570.       {
  571.        PrintMessage(Warning,"The requested method '%s' is not supported for the %s protocol.",request_head->method,Url->Protocol->name);
  572.        if(mode==Fetch)
  573.          {
  574.           if(client!=-1)
  575.              write_formatted(client,"Cannot fetch %s [The %s method is not supported for %s]n",Url->name,request_head->method,Url->Protocol->name);
  576.           exit(1);
  577.          }
  578.        else
  579.          {
  580.           HTMLMessage(tmpclient,501,"WWWOFFLE Method Unsupported",NULL,"MethodUnsupported",
  581.                       "method",request_head->method,
  582.                       "protocol",Url->Protocol->name,
  583.                       NULL);
  584.           mode=SpoolInternal; goto spoolinternal;
  585.          }
  586.       }
  587.    }
  588.  else if(mode==Fetch)
  589.     ParseRecurseOptions(NULL);
  591.  /* mode = Spool, SpoolGet, SpoolRefresh, Real, RealRefresh, SpoolOrReal or Fetch */
  593.  /* Check for a username / password */
  595.  if(Url->user)
  596.    {
  597.     URL *new=SplitURL(Url->name);
  599.     Urlpw=Url;
  600.     Url=new;
  602.     if(!Urlpw->pass)
  603.       {
  604.        if(mode==Fetch)
  605.          {
  606.           FreeURL(Urlpw);
  607.           Urlpw=NULL;
  608.          }
  609.        else
  610.          {
  611.           HTMLMessage(tmpclient,403,"WWWOFFLE Username Needs Password",NULL,"UserNeedsPass",
  612.                       "url",Urlpw->name,
  613.                       "user",Urlpw->user,
  614.                       NULL);
  615.           mode=SpoolInternal; goto spoolinternal;
  616.          }
  617.       }
  618.    }
  620.  /* Check for an existing cached version */
  622.  outgoing_exists=ExistsOutgoingSpoolFile(Url);
  623.  spool_exists=ExistsWebpageSpoolFile(Url);
  625.  if(Urlpw)
  626.    {
  627.     spool_exists_pw=ExistsWebpageSpoolFile(Urlpw);
  629.     if((mode==Spool || mode==SpoolGet || mode==SpoolOrReal) && spool_exists)
  630.       {
  631.        if(SpooledPageStatus(Url)==401)
  632.          {FreeURL(Url);Url=Urlpw;Urlpw=NULL;
  633.          spool_exists=spool_exists_pw;spool_exists_pw=0;}
  634.        else
  635.          {FreeURL(Urlpw);Urlpw=NULL;
  636.          spool_exists_pw=0;}
  637.       }
  638.     else if(mode==Spool || mode==SpoolGet || mode==SpoolOrReal) /* && ! spool_exists */
  639.       {
  640.        if(spool_exists_pw)
  641.          {
  642.           int new_outgoing=OpenOutgoingSpoolFile(0);
  644.           if(new_outgoing==-1)
  645.              PrintMessage(Warning,"Cannot open the new outgoing request to write.");
  646.           else
  647.             {
  648.              char *head=HeaderString(request_head);
  649.              write_string(new_outgoing,head);
  650.              if(request_body)
  651.                 write_data(new_outgoing,request_body->content,request_body->length);
  652.              CloseOutgoingSpoolFile(new_outgoing,Url);
  653.              free(head);
  654.             }
  655.          }
  656.        FreeURL(Url);Url=Urlpw;Urlpw=NULL;
  657.        spool_exists=spool_exists_pw;spool_exists_pw=0;
  658.       }
  659.     else if((mode==Fetch || mode==Real) && spool_exists)
  660.       {
  661.        if(SpooledPageStatus(Url)==401)
  662.          {FreeURL(Url);Url=Urlpw;Urlpw=NULL;
  663.          spool_exists=spool_exists_pw;spool_exists_pw=0;}
  664.        else
  665.          {FreeURL(Urlpw);Urlpw=NULL;
  666.          spool_exists_pw=0;}
  667.       }
  668.    }
  670.  /* If mode is Real or Fetch and a password is set then get page with
  671.     no password then come here and try again with a password. */
  672. passwordagain:
  674.  /* Check if it needs to be cached. */
  676.  if(IsLocalNetHost(Url->host))
  677.    {
  678.     if(mode==Real || mode==Spool || mode==SpoolOrReal)
  679.        mode=RealNoCache;
  680.     else if(mode==Fetch)
  681.       {
  682.        PrintMessage(Inform,"The request to fetch a page from the local network host '%s' is ignored.",Url->host);
  683.        if(client!=-1)
  684.           write_formatted(client,"Cannot fetch %s [On local network]n",Url->name);
  685.        exit(1);
  686.       }
  687.    }
  688.  else if((mode==Real || mode==SpoolOrReal) && IsNotCached(Url->proto,Url->host,Url->path,Url->args))
  689.    {
  690.     mode=RealNoCache;
  691.    }
  693.  if(mode==RealNoCache && Urlpw)
  694.    {FreeURL(Url);Url=Urlpw;Urlpw=NULL;
  695.    spool_exists=spool_exists_pw;spool_exists_pw=0;}
  697.  /* mode = Spool, SpoolGet, SpoolRefresh, Real, RealRefresh, RealNoCache, SpoolOrReal or Fetch */
  699.  /* Check if it was a POST/PUT method. */
  701.  if(!strcmp(request_head->method,"POST") ||
  702.     !strcmp(request_head->method,"PUT"))
  703.    {
  704.     if(mode==Spool)
  705.        mode=SpoolGet;
  706.    }
  707.  else if(Url->args && *Url->args=='!')
  708.    {
  709.     if(mode==Fetch)
  710.       {
  711.        PrintMessage(Inform,"It is not possible to fetch a URL that used the POST/PUT method.");
  712.        if(client!=-1)
  713.           write_formatted(client,"Cannot fetch %s [Reply from a POST/PUT method]n",Url->name);
  714.        exit(1);
  715.       }
  716.     else if(!spool_exists && !outgoing_exists)
  717.       {
  718.        PrintMessage(Inform,"It is not possible to request a URL that used the POST/PUT method.");
  719.        HTMLMessage(tmpclient,404,"WWWOFFLE Cant Refresh POST/PUT",NULL,"CantRefreshPosted",
  720.                    "url",Url->name,
  721.                    NULL);
  722.        mode=SpoolInternal; goto spoolinternal;
  723.       }
  724.     else
  725.        mode=Spool;
  726.    }
  728.  /* Check if it is a conditional request. */
  730.  if(mode!=RealNoCache && GetHeader(request_head,"If-Modified-Since",NULL))
  731.    {
  732.     char *val=GetHeader(request_head,"If-Modified-Since",NULL);
  734.     if(mode==Spool || mode==SpoolOrReal)
  735.       {
  736.        spool=OpenWebpageSpoolFile(1,Url);
  738.        if(spool!=-1)
  739.          {
  740.           Header *spooled_head=NULL;
  741.           char *modified;
  742.           time_t modtime;
  743.           time_t since=DateToTimeT(val);
  745.           init_buffer(spool);
  746.           ParseReply(spool,NULL,&spooled_head);
  748.           if(spooled_head && (modified=GetHeader(spooled_head,"Last-Modified",NULL)))
  749.              modtime=DateToTimeT(modified);
  750.           else
  751.             {
  752.              struct stat buf;
  754.              if(fstat(spool,&buf))
  755.                 modtime=time(NULL)+1;
  756.              else
  757.                 modtime=buf.st_mtime;
  758.             }
  760.           close(spool);
  761.           spool=-1;
  763.           if(spooled_head)
  764.              FreeHeader(spooled_head);
  766.           if(since>=modtime)
  767.             {
  768.              HTMLMessageHead(tmpclient,304,"WWWOFFLE Not Modified",
  769.                              NULL);
  770.              mode=SpoolInternal; goto spoolinternal;
  771.             }
  772.          }
  773.       }
  775.     RemoveFromHeader(request_head,"If-Modified-Since",NULL);
  776.    }
  778.  /* Check if a refresh is needed based on pragma, request changes, autodial. */
  780.  if(mode==Real && ExistsLockWebpageSpoolFile(Url))
  781.    {
  782.     mode=Spool;
  783.    }
  784.  else if(mode==Fetch && ExistsLockWebpageSpoolFile(Url))
  785.    {
  786.     PrintMessage(Debug,"Already fetching URL.");
  787.     if(client!=-1)
  788.        write_formatted(client,"Cannot fetch %s [Already fetching]n",Url->name);
  789.     exit(fetch_again?4:0);
  790.    }
  791.  else if(RefreshForced() || (PragmaNoCache && GetHeader(request_head,"Pragma","no-cache")))
  792.    {
  793.     if(mode==Spool || mode==SpoolGet)
  794.       {
  795.        if(spool_exists)
  796.           mode=SpoolPragma;
  797.        else
  798.           if(outgoing_exists)
  799.              mode=SpoolWillGet;
  800.           else
  801.              mode=SpoolGet;
  802.       }
  803.     else if(mode==SpoolOrReal)
  804.        mode=Real;
  806.     /* (mode==Fetch || mode==Real) are left unchanged, not modified as below. */
  807.    }
  808.  else if(mode==Spool && !spool_exists)
  809.    {
  810.     if(outgoing_exists)
  811.        mode=SpoolWillGet;
  812.     else
  813.        mode=SpoolGet;
  814.    }
  815.  else if(mode==Fetch && spool_exists)
  816.    {
  817.     spool=OpenWebpageSpoolFile(1,Url);
  818.     init_buffer(spool);
  820.     if((RequestChanged>=0 || RequestChangedOnce) && RequestChanges(spool,request_head)==1)
  821.        TouchWebpageSpoolFile(Url,0);
  822.     else if(fetch_again)
  823.       {
  824.        lseek(spool,0,SEEK_SET);
  825.        init_buffer(spool);
  827.        if(ParseDocument(spool,Url))
  828.           RecurseFetch(Url,0);
  830.        exit(4);
  831.       }
  832.     else
  833.        exit(0);
  835.     close(spool);
  836.    }
  837.  else if(mode==Real && spool_exists)
  838.    {
  839.     spool=OpenWebpageSpoolFile(1,Url);
  840.     init_buffer(spool);
  842.     if((RequestChanged>=0 || RequestChangedOnce) && RequestChanges(spool,request_head)==1)
  843.        TouchWebpageSpoolFile(Url,0);
  844.     else
  845.        mode=Spool;
  847.     close(spool);
  848.    }
  849.  else if(mode==SpoolOrReal)
  850.    {
  851.     if(spool_exists)
  852.        mode=Spool;
  853.     else
  854.        mode=Real;
  855.    }
  857.  /* Don't let htdig request any URLs. */
  859.  is_client_htdig=GetHeader(request_head,"User-Agent","htdig/");
  861.  if((mode==SpoolGet || mode==Real) && is_client_htdig)
  862.    {
  863.     PrintMessage(Inform,"URL unavailable to be searched.");
  864.     HTMLMessageHead(tmpclient,404,"WWWOFFLE Not Searched",
  865.                     NULL);
  866.     mode=SpoolInternal; goto spoolinternal;
  867.    }
  869.  /* mode = Spool, SpoolGet, SpoolWillGet, SpoolPragma, SpoolRefresh, Real, RealRefresh, RealNoCache or Fetch */
  871.  /* Set up the file descriptor for the spool file. */
  873.  if(mode==Real || mode==Fetch)
  874.    {
  875.     if(spool_exists)
  876.        CreateBackupWebpageSpoolFile(Url);
  878.     spool=OpenWebpageSpoolFile(0,Url);
  879.     CreateLockWebpageSpoolFile(Url);
  881.     if(spool==-1)
  882.       {
  883.        DeleteLockWebpageSpoolFile(Url);
  884.        PrintMessage(Warning,"Cannot open the spooled web page to write.");
  885.        if(mode==Real)
  886.          {
  887.           HTMLMessage(tmpclient,500,"WWWOFFLE Server Error",NULL,"ServerError",
  888.                       "error","Cannot open the spooled web page to write.",
  889.                       NULL);
  890.           mode=SpoolInternal; goto spoolinternal;
  891.          }
  892.        else /* mode==Fetch */
  893.          {
  894.           if(client!=-1)
  895.              write_formatted(client,"Internal Error %s [Cannot open spool file]n",Url->name);
  896.           exit(1);
  897.          }
  898.       }
  900.     lasttime_exists=CreateLastTimeSpoolFile(Url);
  901.    }
  902.  else if(mode==Spool || mode==SpoolPragma)
  903.    {
  904.     spool=OpenWebpageSpoolFile(1,Url);
  905.     init_buffer(spool);
  907.     if(spool==-1)
  908.       {
  909.        PrintMessage(Warning,"Cannot open the spooled web page to read.");
  910.        HTMLMessage(tmpclient,500,"WWWOFFLE Server Error",NULL,"ServerError",
  911.                    "error","Cannot open the spooled web page to read.",
  912.                    NULL);
  913.        mode=SpoolInternal; goto spoolinternal;
  914.       }
  915.    }
  917.  /* Set up the outgoing file. */
  919.  offline_request=!IsNotRequestedOffline(Url->proto,Url->host,Url->path,Url->args);
  921.  is_client_wwwoffle=GetHeader(request_head,"Pragma","wwwoffle");
  923.  if((offline_request || online) &&
  924.     ((mode==SpoolGet && (!ConfirmRequests || is_client_wwwoffle || Url->local || (Url->args && *Url->args=='!'))) ||
  925.      mode==SpoolRefresh ||
  926.      mode==SpoolPragma))
  927.    {
  928.     outgoing=OpenOutgoingSpoolFile(0);
  930.     if(outgoing==-1)
  931.       {
  932.        PrintMessage(Warning,"Cannot open the outgoing request to write.");
  933.        HTMLMessage(tmpclient,500,"WWWOFFLE Server Error",NULL,"ServerError",
  934.                    "error","Cannot open the outgoing request to write.",
  935.                    NULL);
  936.        mode=SpoolInternal; goto spoolinternal;
  937.       }
  938.    }
  940.  /* Open the connection to the server host. */
  942.  if(mode==Real || mode==RealNoCache || mode==Fetch)
  943.    {
  944.     char *err=(Url->Protocol->open)(Url);
  946.     if(err && ConnectRetry)
  947.       {
  948.        PrintMessage(Inform,"Waiting to try connection again.");
  949.        sleep(10);
  950.        err=(Url->Protocol->open)(Url);
  951.       }
  953.     if(err)
  954.       {
  955.        if(mode==Fetch || mode==Real)
  956.          {
  957.           lseek(spool,0,SEEK_SET);
  958.           ftruncate(spool,0);
  959.           HTMLMessage(spool,503,"WWWOFFLE Remote Host Error",NULL,"RemoteHostError",
  960.                       "url",Url->name,
  961.                       "reason",err,
  962.                       "cache","yes",
  963.                       "backup",spool_exists?"yes":NULL,
  964.                       NULL);
  965.           DeleteLockWebpageSpoolFile(Url);
  966.           close(spool);
  967.          }
  968.        if(mode==Fetch)
  969.          {
  970.           if(client!=-1)
  971.              write_formatted(client,"Fetch Failure %s [Server Connection Failed]n",Url->name);
  972.           exit(1);
  973.          }
  974.        else
  975.          {
  976.           HTMLMessage(tmpclient,503,"WWWOFFLE Remote Host Error",NULL,"RemoteHostError",
  977.                       "url",Url->name,
  978.                       "reason",err,
  979.                       "cache","",
  980.                       "backup",spool_exists?"yes":NULL,
  981.                       NULL);
  982.           mode=SpoolInternal; goto spoolinternal;
  983.          }
  984.       }
  985.    }
  987.  /* Modify the request header (Censor / Cannonicalise URL / POST/PUT / HTTP-1.1 etc). */
  989.  if(mode==Real || mode==RealNoCache || mode==Fetch || mode==SpoolGet || mode==SpoolRefresh || mode==SpoolPragma)
  990.     ModifyRequest(Url,request_head);
  992.  /* Write request to remote server or outgoing file. */
  994.  if(mode==Fetch && client!=-1)
  995.     write_formatted(client,"Fetching %s ...n",Url->name);
  997.  if(mode==Real || mode==RealNoCache || mode==Fetch)
  998.    {
  999.     char *err=(Url->Protocol->request)(Url,request_head,request_body);
  1001.     if(err)
  1002.       {
  1003.        if(mode==Real || mode==Fetch)
  1004.          {
  1005.           lseek(spool,0,SEEK_SET);
  1006.           ftruncate(spool,0);
  1007.           HTMLMessage(spool,503,"WWWOFFLE Remote Host Error",NULL,"RemoteHostError",
  1008.                       "url",Url->name,
  1009.                       "reason",err,
  1010.                       "cache","yes",
  1011.                       "backup",spool_exists?"yes":NULL,
  1012.                       NULL);
  1013.           DeleteLockWebpageSpoolFile(Url);
  1014.          }
  1015.        if(mode==Real || mode==RealNoCache)
  1016.          {
  1017.           HTMLMessage(tmpclient,503,"WWWOFFLE Remote Host Error",NULL,"RemoteHostError",
  1018.                       "url",Url->name,
  1019.                       "reason",err,
  1020.                       "cache",NULL,
  1021.                       "backup",spool_exists?"yes":NULL,
  1022.                       NULL);
  1023.           mode=SpoolInternal; goto spoolinternal;
  1024.          }
  1025.        else /* mode==Fetch */
  1026.          {
  1027.           if(client!=-1)
  1028.              write_formatted(client,"Fetch Failure %s [Server Connection Error]n",Url->name);
  1029.           exit(1);
  1030.          }
  1031.       }
  1032.    }
  1033.  else if((offline_request || online) &&
  1034.          ((mode==SpoolGet && (!ConfirmRequests || Url->local || (Url->args && *Url->args=='!'))) ||
  1035.           mode==SpoolRefresh ||
  1036.           mode==SpoolPragma))
  1037.    {
  1038.     char *head=HeaderString(request_head);
  1039.     int err=write_string(outgoing,head);
  1041.     if(err==-1)
  1042.       {
  1043.        PrintMessage(Warning,"Cannot write the outgoing request.");
  1044.        HTMLMessage(tmpclient,500,"WWWOFFLE Server Error",NULL,"ServerError",
  1045.                    "error","Cannot write the outgoing request.",
  1046.                    NULL);
  1047.        mode=SpoolInternal; goto spoolinternal;
  1048.       }
  1050.     if(request_body)
  1051.        write_data(outgoing,request_body->content,request_body->length);
  1053.     free(head);
  1054.    }
  1056.  /* Parse the reply */
  1058.  if(mode==Real || mode==RealNoCache || mode==Fetch)
  1059.    {
  1060.     reply_status=ParseReply(-1,Url,&reply_head);
  1062.     if(!reply_head)
  1063.       {
  1064.        PrintMessage(Warning,"Timed out reading the reply.");
  1065.        if(mode==Real || mode==Fetch)
  1066.          {
  1067.           lseek(spool,0,SEEK_SET);
  1068.           ftruncate(spool,0);
  1069.           HTMLMessage(spool,503,"WWWOFFLE Remote Host Error",NULL,"RemoteHostError",
  1070.                       "url",Url->name,
  1071.                       "reason","TimeoutReply",
  1072.                       "cache","yes",
  1073.                       "backup",spool_exists?"yes":NULL,
  1074.                       NULL);
  1075.           DeleteLockWebpageSpoolFile(Url);
  1076.          }
  1077.        if(mode==Real || mode==RealNoCache)
  1078.           HTMLMessage(client,503,"WWWOFFLE Remote Host Error",NULL,"RemoteHostError",
  1079.                       "url",Url->name,
  1080.                       "reason","TimeoutReply",
  1081.                       "cache",NULL,
  1082.                       "backup",spool_exists?"yes":NULL,
  1083.                       NULL);
  1084.        else if(mode==Fetch && client!=-1)
  1085.           write_formatted(client,"Fetch Failure %s [Server Reply Error]n",Url->name);
  1086.        exit(1);
  1087.       }
  1089.     if(DebugLevel==ExtraDebug)
  1090.        PrintMessage(ExtraDebug,"Incoming Reply Head (from server/proxy)n%s",HeaderString(reply_head));
  1092.     if(mode==Fetch && (reply_status==301 || reply_status==302))
  1093.       {
  1094.        char *new_url=MovedLocation(Url,reply_head);
  1096.        if(client!=-1)
  1097.           write_formatted(client,"Fetching More %s [Page Moved]n",Url->name);
  1099.        if(!new_url)
  1100.           PrintMessage(Warning,"Cannot parse the reply for the new location.");
  1101.        else
  1102.           fetch_again+=RecurseFetchRelocation(Url,new_url);
  1103.       }
  1104.     else if(reply_status==304)
  1105.       {
  1106.        PrintMessage(Inform,"Server page is not newer than the one in cache.");
  1108.        if(mode==Fetch || mode==Real)
  1109.          {
  1110.           close(spool);
  1111.           DeleteLockWebpageSpoolFile(Url);
  1112.           RestoreBackupWebpageSpoolFile(Url);
  1113.           if(!lasttime_exists)
  1114.              DeleteLastTimeSpoolFile(Url);
  1115.          }
  1116.        if(mode==Fetch)
  1117.          {
  1118.           if(client!=-1)
  1119.              write_formatted(client,"Not fetching %s [Page Unchanged]n",Url->name);
  1121.           if(fetch_again)
  1122.             {
  1123.              spool=OpenWebpageSpoolFile(1,Url);
  1124.              init_buffer(spool);
  1126.              if(ParseDocument(spool,Url))
  1127.                 RecurseFetch(Url,1);
  1129.              close(spool);
  1130.             }
  1132.           exit(fetch_again?4:0);
  1133.          }
  1134.        else if(mode==Real)
  1135.          {
  1136.           spool=OpenWebpageSpoolFile(1,Url);
  1137.           init_buffer(spool);
  1139.           mode=Spool;
  1140.          }
  1141.       }
  1142.     else if(mode==Fetch && reply_status==401 && Urlpw)
  1143.       {
  1144.        mode=FetchNoPassword;
  1145.        if(client!=-1)
  1146.           write_formatted(client,"Fetching More %s [URL with password]n",Url->name);
  1147.       }
  1148.     else if(mode==Real && reply_status==401 && Urlpw)
  1149.       {
  1150.        mode=RealNoPassword;
  1151.       }
  1152.    }
  1153.  else
  1154.     reply_head=NULL;
  1156.  if(mode==Spool || mode==SpoolPragma)
  1157.    {
  1158.     reply_status=ParseReply(spool,Url,&reply_head);
  1160.     if(!reply_head)
  1161.        PrintMessage(Warning,"Spooled Reply Head (from cache) is empty.");
  1162.     else
  1163.       {
  1164.        char *head=HeaderString(reply_head);
  1165.        if(DebugLevel==ExtraDebug)
  1166.           PrintMessage(ExtraDebug,"Spooled Reply Head (from cache)n%s",head);
  1167.        free(head);
  1168.       }
  1169.    }
  1171.  reply_body=CreateBody(READ_BUFFER_SIZE);
  1173.  /* mode = Spool, SpoolGet, SpoolWillGet, SpoolPragma, SpoolRefresh, Real, RealRefresh, RealNoCache, RealNoPassword, Fetch or FetchNoPassword */
  1175.  /* Close the outgoing file if any. */
  1177.  if(outgoing>=0)
  1178.    {
  1179.     if(mode==Fetch)
  1180.        close(outgoing);
  1181.     if(mode==SpoolGet || mode==SpoolRefresh || mode==SpoolPragma)
  1182.        CloseOutgoingSpoolFile(outgoing,Url);
  1183.    }
  1185.  /* The main body of the handling of the data. */
  1187.  if(mode==Real || mode==RealNoCache || mode==RealNoPassword)
  1188.    {
  1189.     char *head;
  1190.     int n=0,err=0,bytes=0;
  1192.     ModifyReply(reply_head);
  1194.     head=HeaderString(reply_head);
  1196.     if(DebugLevel==ExtraDebug)
  1197.        PrintMessage(ExtraDebug,"Outgoing Reply Head (to browser)n%s",head);
  1199.     if(mode!=RealNoCache)
  1200.        write_string(spool,head);
  1201.     if(mode!=RealNoPassword)
  1202.        err=write_string(client,head);
  1204.     free(head);
  1206.     while(err!=-1 && (n=(Url->Protocol->readbody)(reply_body->content,READ_BUFFER_SIZE))>0)
  1207.       {
  1208.        bytes+=n;
  1210.        if(mode!=RealNoCache)
  1211.           write_data(spool,reply_body->content,n);
  1212.        if(mode!=RealNoPassword && !head_only)
  1213.           err=write_data(client,reply_body->content,n);
  1215.        if(err==-1)
  1216.          {
  1217.           char *length=GetHeader(reply_head,"Content-Length",NULL);
  1219.           if(length)
  1220.             {
  1221.              int size=atoi(length);
  1222.              if(size<(IntrDownloadSize<<10) || (100*(double)bytes/(double)size)>IntrDownloadPercent)
  1223.                 err=0;
  1224.              head_only=1; /* cheat to stop writing to client */
  1225.             }
  1226.          }
  1227.       }
  1229.     if(mode!=RealNoCache)
  1230.       {
  1231.        if(err==-1 && !IntrDownloadKeep)
  1232.          {
  1233.           PrintMessage(Warning,"Error writing to client [%!s]; client disconnected?");
  1234.           lseek(spool,0,SEEK_SET);
  1235.           ftruncate(spool,0);
  1236.           HTMLMessage(spool,503,"WWWOFFLE Remote Host Error",NULL,"RemoteHostError",
  1237.                       "url",Url->name,
  1238.                       "reason","ClientClose",
  1239.                       "cache","yes",
  1240.                       "backup",spool_exists?"yes":NULL,
  1241.                       NULL);
  1242.          }
  1243.        else if(n<0 && !TimeoutDownloadKeep)
  1244.          {
  1245.           PrintMessage(Warning,"Timed out while reading from remote host.");
  1246.           lseek(spool,0,SEEK_SET);
  1247.           ftruncate(spool,0);
  1248.           HTMLMessage(spool,503,"WWWOFFLE Remote Host Error",NULL,"RemoteHostError",
  1249.                       "url",Url->name,
  1250.                       "reason","TimeoutTransfer",
  1251.                       "cache","yes",
  1252.                       "backup",spool_exists?"yes":NULL,
  1253.                       NULL);
  1254.          }
  1255.        else
  1256.          {
  1257.           if(n<0 || err==-1)
  1258.             {
  1259.              reply_head->status=503;
  1260.              head=HeaderString(reply_head);
  1261.              lseek(spool,0,SEEK_SET);
  1262.              write_string(spool,head);
  1263.              free(head);
  1264.             }
  1266.           if(spool_exists)
  1267.              DeleteBackupWebpageSpoolFile(Url);
  1268.          }
  1270.        DeleteLockWebpageSpoolFile(Url);
  1271.       }
  1272.    }
  1273.  else if(mode==Fetch || mode==FetchNoPassword)
  1274.    {
  1275.     char *head=HeaderString(reply_head);
  1276.     int n;
  1278.     write_string(spool,head);
  1280.     free(head);
  1282.     while((n=(Url->Protocol->readbody)(reply_body->content,READ_BUFFER_SIZE))>0)
  1283.        write_data(spool,reply_body->content,n);
  1285.     if(n<0 && !TimeoutDownloadKeep)
  1286.       {
  1287.        PrintMessage(Warning,"Timed out while reading from remote host.");
  1288.        lseek(spool,0,SEEK_SET);
  1289.        ftruncate(spool,0);
  1290.        HTMLMessage(spool,503,"WWWOFFLE Remote Host Error",NULL,"RemoteHostError",
  1291.                    "url",Url->name,
  1292.                    "reason","TimeoutTransfer",
  1293.                    "cache","yes",
  1294.                    "backup",spool_exists?"yes":NULL,
  1295.                    NULL);
  1297.        if(client!=-1)
  1298.           write_formatted(client,"Fetch Failure %s [Timeout]n",Url->name);
  1299.       }
  1300.     else
  1301.       {
  1302.        if(spool_exists)
  1303.           DeleteBackupWebpageSpoolFile(Url);
  1304.        if(client!=-1)
  1305.           write_formatted(client,"Fetch Success %sn",Url->name);
  1306.       }
  1308.     DeleteLockWebpageSpoolFile(Url);
  1310.     if(n>=0 && reply_status>=200 && reply_status<400)
  1311.       {
  1312.        lseek(spool,0,SEEK_SET);
  1313.        init_buffer(spool);
  1315.        if(ParseDocument(spool,Url))
  1316.          {
  1317.           int links=RecurseFetch(Url,1);
  1319.           if(client!=-1 && links)
  1320.              write_formatted(client,"Fetching More %s [%d Extra URLs]n",Url->name,links);
  1322.           fetch_again+=links;
  1323.          }
  1324.       }
  1325.    }
  1326.  else if(mode==Spool || mode==SpoolPragma)
  1327.    {
  1328.     struct stat buf;
  1329.     char *remote_error_note="WWWOFFLE Remote Host Error";
  1331.     if(ExistsLockWebpageSpoolFile(Url))
  1332.       {
  1333.        int t=0;
  1335.        if(online)
  1336.          {
  1337.           t=SocketTimeout/6;
  1339.           PrintMessage(Inform,"Waiting for the page to be unlocked.");
  1341.           while(--t>0 && ExistsLockWebpageSpoolFile(Url))
  1342.              sleep(1);
  1344.           if(t<=0)
  1345.              PrintMessage(Inform,"Timed out waiting for the page to be unlocked.");
  1346.          }
  1348.        if(t<=0)
  1349.          {
  1350.           HTMLMessage(tmpclient,503,"WWWOFFLE File Locked",NULL,"FileLocked",
  1351.                       "url",Url->name,
  1352.                       NULL);
  1353.           mode=SpoolInternal; goto spoolinternal;
  1354.          }
  1355.       }
  1357.     if((reply_head && !strcmp(reply_head->note,remote_error_note)) || (!fstat(spool,&buf) && buf.st_size==0))
  1358.       {
  1359.        DeleteWebpageSpoolFile(Url,0);
  1360.        RestoreBackupWebpageSpoolFile(Url);
  1361.       }
  1363.     if(reply_head)
  1364.       {
  1365.        char *head=HeaderString(reply_head);
  1366.        write_string(tmpclient,head);
  1367.        free(head);
  1368.       }
  1370.     if(EnableHTMLModifications &&
  1371.        reply_status>=200 && reply_status<400 &&
  1372.        GetHeader(reply_head,"Content-Type","text/html") &&
  1373.        !is_client_wwwoffle &&
  1374.        !is_client_htdig)
  1375.        OutputHTMLWithModifications(tmpclient,spool,Url);
  1376.     else if(DisableAnimatedGIF &&
  1377.        reply_status>=200 && reply_status<400 &&
  1378.        GetHeader(reply_head,"Content-Type","image/gif") &&
  1379.        !is_client_wwwoffle &&
  1380.        !is_client_htdig)
  1381.        OutputGIFWithModifications(tmpclient,spool,Url);
  1382.     else
  1383.       {
  1384.        int n;
  1386.        while((n=read_data(spool,reply_body->content,READ_BUFFER_SIZE))>0)
  1387.           write_data(tmpclient,reply_body->content,n);
  1388.       }
  1390.     mode=SpoolInternal;
  1391.    }
  1392.  else if(mode==SpoolGet)
  1393.    {
  1394.     if((offline_request || online))
  1395.       {
  1396.        if(ConfirmRequests && !Url->local && (!Url->args || *Url->args!='!'))
  1397.           HTMLMessage(tmpclient,404,"WWWOFFLE Confirm Request",NULL,"ConfirmRequest",
  1398.                       "url",Url->name,
  1399.                       NULL);
  1400.        else if(fetch_again)
  1401.           HTMLMessage(tmpclient,404,"WWWOFFLE Refresh Will Get",NULL,"RefreshWillGet",
  1402.                       "url",Url->name,
  1403.                       "password",HashOutgoingSpoolFile(Url),
  1404.                       NULL);
  1405.        else
  1406.           HTMLMessage(tmpclient,404,"WWWOFFLE Will Get",NULL,"WillGet",
  1407.                       "already",NULL,
  1408.                       "url",Url->name,
  1409.                       "password",HashOutgoingSpoolFile(Url),
  1410.                       NULL);
  1411.       }
  1412.     else
  1413.        HTMLMessage(tmpclient,404,"WWWOFFLE Refused Request",NULL,"RefusedRequest",
  1414.                    "url",Url->name,
  1415.                    NULL);
  1417.     mode=SpoolInternal;
  1418.    }
  1419.  else if(mode==SpoolWillGet)
  1420.    {
  1421.     HTMLMessage(tmpclient,404,"WWWOFFLE Will Get",NULL,"WillGet",
  1422.                 "already","yes",
  1423.                 "url",Url->name,
  1424.                 "password",HashOutgoingSpoolFile(Url),
  1425.                 NULL);
  1426.     mode=SpoolInternal;
  1427.    }
  1428.  else if(mode==RealRefresh || mode==SpoolRefresh)
  1429.    {
  1430.     HTMLMessage(tmpclient,301,"WWWOFFLE Refresh Redirect",Url->link,"RefreshRedirect",
  1431.                 "url",Url->name,
  1432.                 "link",Url->link,
  1433.                 NULL);
  1434.     mode=SpoolInternal;
  1435.    }
  1437. spoolinternal:                  /* Jump here if mode is SpoolInternal. */
  1439.  if(mode==SpoolInternal)
  1440.    {
  1441.     int n;
  1442.     unsigned long size;
  1444.     if(!reply_body)
  1445.        reply_body=CreateBody(READ_BUFFER_SIZE);
  1447.     size=lseek(tmpclient,0,SEEK_CUR);
  1448.     lseek(tmpclient,0,SEEK_SET);
  1449.     init_buffer(tmpclient);
  1451.     reply_status=ParseReply(tmpclient,Url,&reply_head);
  1453.     if(!reply_head)
  1454.        PrintMessage(Warning,"Outgoing Reply Head (to browser) is empty.");
  1455.     else
  1456.       {
  1457.        char *head,length[10];
  1459.        size-=reply_head->size;
  1461.        RemoveFromHeader(reply_head,"Content-Length",NULL);
  1463.        sprintf(length,"%ld",size);
  1464.        AddToHeader(reply_head,"Content-Length",length);
  1466.        ModifyReply(reply_head);
  1468.        head=HeaderString(reply_head);
  1470.        if(DebugLevel==ExtraDebug)
  1471.           PrintMessage(ExtraDebug,"Outgoing Reply Head (to browser)n%s",head);
  1472.        write_string(client,head);
  1474.        free(head);
  1475.       }
  1477.     if(!head_only)
  1478.        while((n=read_data(tmpclient,reply_body->content,READ_BUFFER_SIZE))>0)
  1479.           write_data(client,reply_body->content,n);
  1480.    }
  1482.  /* Close down and exit. */
  1484.  if((mode==Real || mode==RealNoPassword) && outgoing_exists)
  1485.     DeleteOutgoingSpoolFile(Url);
  1487.  if(mode==Real || mode==RealNoCache || mode==RealNoPassword ||
  1488.     mode==Fetch || mode==FetchNoPassword)
  1489.     (Url->Protocol->close)();
  1491.  if(tmpclient>=0)
  1492.     CloseTempSpoolFile(tmpclient);
  1494.  if(spool>=0)
  1495.     close(spool);
  1497.  if(mode==SpoolInternal && request_head && spool_exists && is_client_htdig)
  1498.     TouchWebpageSpoolFile(Url,spool_exists);
  1500.  if(reply_head)
  1501.     FreeHeader(reply_head);
  1502.  if(reply_body)
  1503.     FreeBody(reply_body);
  1505.  if(mode==RealNoPassword || mode==FetchNoPassword)
  1506.    {
  1507.     FreeURL(Url);Url=Urlpw;Urlpw=NULL;
  1508.     spool_exists=spool_exists_pw;spool_exists_pw=0;
  1509.     reply_head=NULL;
  1510.     reply_body=NULL;
  1511.     if(mode==RealNoPassword)
  1512.        mode=Real;
  1513.     else
  1514.        mode=Fetch;
  1515.     goto passwordagain;
  1516.    }
  1518.  if(request_head)
  1519.     FreeHeader(request_head);
  1520.  if(request_body)
  1521.     FreeBody(request_body);
  1523.  if(client>=0)
  1524.     CloseSocket(client);
  1526.  if(fetch_again)
  1527.     return(4);
  1528.  else
  1529.     return(0);
  1530. }
  1533. /*++++++++++++++++++++++++++++++++++++++
  1534.   Make an SSL proxy connection.
  1536.   int ssl_tunnel Return 1 in case of failure.
  1538.   int client The client socket.
  1540.   URL *Url The URL to get (used for host only).
  1542.   Header *request_head The head of the request.
  1543.   ++++++++++++++++++++++++++++++++++++++*/
  1545. static int ssl_tunnel(int client,URL *Url,Header *request_head)
  1546. {
  1547.  char *err=SSL_Open(Url);
  1549.  if(err && ConnectRetry)
  1550.    {
  1551.     PrintMessage(Inform,"Waiting to try connection again.");
  1552.     sleep(10);
  1553.     err=SSL_Open(Url);
  1554.    }
  1556.  if(err)
  1557.    {
  1558.     HTMLMessage(client,503,"WWWOFFLE Remote Host Error",NULL,"RemoteHostError",
  1559.                 "url",Url->host,
  1560.                 "reason",err,
  1561.                 "cache",NULL,
  1562.                 "backup",NULL,
  1563.                 NULL);
  1564.     return(1);
  1565.    }
  1567.  ModifyRequest(Url,request_head);
  1569.  err=SSL_Request(client,Url,request_head);
  1571.  if(err)
  1572.    {
  1573.     HTMLMessage(client,503,"WWWOFFLE Remote Host Error",NULL,"RemoteHostError",
  1574.                 "url",Url->host,
  1575.                 "reason",err,
  1576.                 "cache",NULL,
  1577.                 "backup",NULL,
  1578.                 NULL);
  1579.     return(1);
  1580.    }
  1581.  else if(!SSLProxy)
  1582.     HTMLMessageHead(client,200,"WWWOFFLE SSL OK",
  1583.                     "Content-Type",NULL,
  1584.                     NULL);
  1586.  SSL_Transfer(client);
  1588.  SSL_Close();
  1590.  return(0);
  1591. }
  1594. /*++++++++++++++++++++++++++++++++++++++
  1595.   Uninstall the signal handlers.
  1596.   ++++++++++++++++++++++++++++++++++++++*/
  1598. static void uninstall_sighandlers(void)
  1599. {
  1600.  struct sigaction action;
  1602.  /* SIGCHLD */
  1603.  action.sa_handler = SIG_DFL;
  1604.  sigemptyset (&action.sa_mask);
  1605.  action.sa_flags = 0;
  1606.  if(sigaction(SIGCHLD, &action, NULL) != 0)
  1607.     PrintMessage(Warning, "Cannot uninstall SIGCHLD handler.");
  1609.  /* SIGINT, SIGQUIT, SIGTERM */
  1610.  action.sa_handler = SIG_DFL;
  1611.  sigemptyset(&action.sa_mask);
  1612.  action.sa_flags = 0;
  1613.  if(sigaction(SIGINT, &action, NULL) != 0)
  1614.     PrintMessage(Warning, "Cannot uninstall SIGINT handler.");
  1615.  if(sigaction(SIGQUIT, &action, NULL) != 0)
  1616.     PrintMessage(Warning, "Cannot uninstall SIGQUIT handler.");
  1617.  if(sigaction(SIGTERM, &action, NULL) != 0)
  1618.     PrintMessage(Warning, "Cannot uninstall SIGTERM handler.");
  1620.  /* SIGHUP */
  1621.  action.sa_handler = SIG_DFL;
  1622.  sigemptyset(&action.sa_mask);
  1623.  action.sa_flags = 0;
  1624.  if(sigaction(SIGHUP, &action, NULL) != 0)
  1625.     PrintMessage(Warning, "Cannot uninstall SIGHUP handler.");
  1627.  /* SIGPIPE */
  1628.  action.sa_handler = SIG_IGN;
  1629.  sigemptyset (&action.sa_mask);
  1630.  action.sa_flags = 0;
  1631.  if(sigaction(SIGPIPE, &action, NULL) != 0)
  1632.     PrintMessage(Warning, "Cannot ignore SIGPIPE.");
  1633. }