#!/iw/perl/current/bin/perl
#
#  file: cbd2dtd
#  desc: Get the final XML output
#

# limit scope
package CBD::cbddtd;

use strict;
use vars qw($VERSION $RELEASE);

#
# current module verion
my $Id =<<'EoI';
#  $Id: //depot/isms/skulker/cbd/1.1/bin/cbd2dtd#4 $
EoI

#
my $RELEASE = sprintf("%d", $Id =~ /^#  \$Id: .*#(\d+)/);
my $VERSION = "1.1";

BEGIN {
        my $iw_root = ($ENV{IW}) ? $ENV{IW} : "/iw";
        my $cbd_root = "$iw_root/skulker/cbd/current";

        # where we find our local libraries
        my($libdir) = "$cbd_root/lib";
        if ( -d $libdir) {
                unshift(@INC, $libdir);
        }

        # CBD LIB support
        require 'cbd-lib.pl';
}


# Get Date parts from a string
sub getDateParts
{
        my($dateStr) = shift;
        my($month, $day, $year);

        my(@Months) = ('', 'January', 'February', 'March', 'April', 'May', 'June',
           'July', 'August', 'September', 'October', 'November', 'December');

        # remove an extra dash or colon in date info
        $dateStr =~ s/-/ /g;
        $dateStr =~ s/:/ /g;
        $dateStr =~ s/'/ /g;
        $dateStr =~ s@</TEXT>@@g;

        # remove extra white space in date info
        $dateStr =~ s@\s{2,}@ @g;

        if ($dateStr =~ /\//) {        
                # month/day/year format
                ($month, $day, $year) = split(/\//, $dateStr);
                # in some cases, no day is given
                if (! $year) {
                        $year = $day;
                        $day = "";
                }
        } elsif ($dateStr =~ /,/) {     # month day, year format
                ($month, $day, $year) = $dateStr =~ /(.*?) (\d{1,2}),\s*(\d{2,4})/;
        } else {                                                           
                ($month, $day, $year) = split(/\s+/, $dateStr);
        }

        # month should be written out, so if a number make the change
        if ($month =~ /\d+/) {
                $month = $Months[$month];
        }

        # remove anything other than numbers from year and day
        $year =~ s/\D//g;
        $day =~ s/\D//g;

        # pad digits to years, giving a 4 number year
        if (length($year) == 2) {
                if ($year > 70) {
                        $year = "19" . $year;
                } else {
                        $year = "20" . $year;
                }
        } elsif ((length($year) == 6) || (length($year) == 5)) {
                ($day, $year) = $year =~ /(\d{1,2})(\d{4})/;
        }

        # pad a zero to the front, if the day is a single number
        if (length($day) == 1) {
                $day = "0" . $day;
        }

        return ($month, $day, $year);
}



# Get Address parts

sub getAddressParts
{
   my($address) = @_;
   my(@country) = ('Liechtenstein', 'Japan');
   my(@states) = ('New Jersey', 'NJ', 'New York', 'NY', 'California', 'CA',
                     'Virginia', 'VA', 'Nebraska', 'NE', 'Florida', 'FL',
                     'Oklahoma', 'OK', 'Massachusetts', 'MA', 'Utah', 'UT',
                     'ILLINOIS', 'IL', 'Kentucky', 'KY', 'Georgia', 'GA',
                     'Alabama', 'AL', 'Connecticutt', 'CT', 'Texas', 'TX',
                     'Pennsylvania', 'PA', 'Delaware', 'DE', 'Mississippi', 'MS',
                     'Minnesota', 'MN', 'Iowa', 'IA', 'Missouri', 'MO',
                     'Wisconsin', 'WI', 'Maryland', 'MD', 'North Carolina', 'NC',
                     'South Carolina', 'SC', 'Indiana', 'IN', 'West Virginia', 'WV',
                     'Ohio', 'OH', 'North Dakota', 'ND', 'South Dakota', 'SD',
                     'Washington', 'WA', 'Oregon', 'OR', 'Arizona', 'AZ', 'Alaska',
                     'AK', 'Vermont', 'VT', 'New Hampshire', 'NH', 'Tennesse', 'TN',
                     'Lousiana', 'LA', 'Kansas', 'KS', 'Arkansas', 'AR', 'Wyoming',
                     'WY', 'Colorado', 'CO', 'Idaho', 'ID', 'Michigan', 'MI',
                     'Nevada', 'NV', 'Hawai', 'HI', 'Maine', 'ME', 'Montana', 'MT',
                     'DC', 'Rhode Island', 'RI', 'New Mexico', 'NM');
   my($statedone) = 0;
   my($state, $st, $street, $city, $country, $zip, $c, $l, $linecount, $index);
   my($office);
   my(@lines);
   $address =~ s/_{3,}//gsi;
   foreach $st(@states)
   {
      if($address =~ /\b$st\b\.*,*\s*\d+/i)
      {
         $state = $st;
         last;
      }
   }

   # Check if international
   my($international);
   foreach $c(@country)
   {
      if($address =~ /$c\W*/)
      {
         $country = $c;
      }
   }

   # United States

   if(length($country) eq 0)
   {
   # Get zip code
   if(length($state) gt 0)
   {
   ($zip) = ($address =~ /.*$state\.*,*\s*(\d+-*\d*)/si);

   # Get city
   ($street) = ($address =~ /(.*)$state.*$zip/si);

   ($street) = ($address =~ /(.*)$city.*$state.*$zip/si);
   @lines = split(/,/, $street);
   $linecount = 0;
   foreach $l(@lines)
   {
      $linecount++;
   }
   for($index=$linecount-1;$index>=0;$index--)
   {
      if($lines[$index] =~ /\w+/)
      {
         $city = $lines[$index];
         last;
      }
   }
   $city =~ s/\)//gsi;
   $city =~ s/\(//gsi;
   $street =~ s/$city//;
   if($street =~ /\w+/)
   {  # Do nothing
   }
   else
   {
      $street = $city;
      $city = "";
   } 
   $office = $street;
   @lines = split(/,/, $office);
   $linecount = 0;
   foreach $l(@lines)
   {
      $linecount++;
   }
   for($index=$linecount-1;$index>=0;$index--)
   {
      if($lines[$index] =~ /\w+/)
      {
         $street = $lines[$index];
         $street =~ s/\(//gsi;
         $street =~ s/\)//gsi;
         last;
      }
   }
   $office =~ s/$street//;
   if($office =~ /\w+/)
   {  # Do nothing
   }
   else
   {
      $office = $street;
      $street = "";
   }
   }
   else { $office = $address; }
   }

   else
   {
      # Get zip code
      ($zip) = ($address =~ /.*(\d*-*\d*)\W*$country/si);

      ($street) = ($address =~ /(.*)$state.*$zip/si);

      ($street) = ($address =~ /(.*).*$zip.*$country/si);
      @lines = split(/,/, $street);
      $linecount = 0;
      foreach $l(@lines)
      {
         $linecount++;
      }
      for($index=$linecount-1;$index>=0;$index--)
      {
         if($lines[$index] =~ /\w+/)
         {
            $city = $lines[$index];
            last;
         }
      }

      $street =~ s/$city//gsi;
      if($street =~ /\w+/)
      {   # Do Nothing 
      }
      else
      {
         $street = $city;
         $city = "";
      }

      $office = $street;

      @lines = split(/,/, $office);
      $linecount = 0;
      foreach $l(@lines)
      {
         $linecount++;
      }
      for($index=$linecount-1;$index>=0;$index--)
      {
         if($lines[$index] =~ /\w+/)
         {
            $street = $lines[$index];
            last;
         }
      }
      
      $office =~ s/$street//gsi;
      if($office =~ /\w+/)
      {  # Do nothing   
      }
      else
      {
         $office = $street;
         $street = "";
      }
   }
   $office =~ s/TO //gsi;
   $street =~ s/$city//gsi;
   $office =~ s/,/ /gsi;
   $street =~ s/,/ /gsi;
   $city =~ s/,/ /gsi;
   $zip =~ s/,/ /gsi;

   return ($office, $street, $city, $state, $zip, $country);

}


# Get Phone numbers
sub getContactPhone
{
   my($in) = @_;
   my($found) = 1;
   my(@phone);
   my($count) = 0;
   while($found)
   {
      if($in =~ /\(?\d\d\d\)?\s*-*\d\d\d\s*-*\d\d\d\d/g)
      {
         ($phone[$count], $in) = ($in =~ /(\(?\d\d\d\)?\s*-*\d\d\d\s*-*\d\d\d\d)(.*)/g);
         $count++;
      }
      else
      {
         $found = 0;
      }
   } 
   return(@phone);
}


# Get Email Addresses
sub getEmailAddress
{
   my($in) = @_;
   my($found) = 1;
   my(@email);
   my($count) = 0;
   while($found)
   {
      if($in =~ /\S+?@\S+/g)
      {
         ($email[$count], $in) = ($in =~ /(\S+?@\S+)/);
         $count++;
      }
      else
      {
         $found = 0;
      }
   }
   return(@email);
}


#
# remove trailing and leading spaces
#
sub trim
{
        my($text) = shift;

        $text =~ s/^\s+|\s+$//g;

        $text;
}



my($XMLout, $input, $firsttag, $endtag, $reqchunk, $day, $month, $year, $date);
my($link, $count, $found, $temp, $pubdate, $maildate, $issue, $type);
my($pubday, $pubdayname, $pubmonth, $pubyear, $mailday, $mailmonth, $mailyear);
my($classcod, $code, $desc, $offadd, $newclasscod, $street, $city, $region, $zip);
my($country, $newoffadd, $subject, $newsubject, $respdate, $newrespdate, $newlink);
my($uri, $email, $newemail, $address, $awarddate, $newawarddate);
my($awardee, $newawardee, $contact, $newcontact, $newdesc, $u, $office);
my(@url, @phone, @personname, @persontitle, @mail);
my(@req, @lines, @contacts);
my($recd, $submitted, $submissionnumber, $origsub, $subpart, $subnum, $solnbr);

my(@Months) = ('', 'January', 'February', 'March', 'April', 'May', 'June',
           'July', 'August', 'September', 'October', 'November', 'December');
my($debug) = 0;

# for all files do
FILE:
foreach my $file(@ARGV)
{
   print STDERR "Processing $file ...\n"
     if ($debug);

   #$XMLout = "<!DOCTYPE cbd SYSTEM \"/data/cbd/etc/cbdspace.dtd\">\n";
   $input = "";

   open(IN, "$file") || (warn "** error reading $file: $!\n", next FILE);
   while(<IN>)
   {
      $input .= $_;
   }

   # Get some header information
   ($issue) = ($input =~ /<ISSNBR>(.*)<\/ISSNBR>/s);
   ($pubdate) = ($input =~ /<PUBDATE>(.*)<\/PUBDATE>/s);
   ($maildate) = ($input =~ /<MAILDATE>(.*)<\/MAILDATE>/s);
   ($pubdayname) = ($pubdate =~ /<DAY>(.*)<\/DAY>/s);
   ($pubdate) = ($pubdate =~ /<DATE>(.*)<\/DATE>/s);
   $maildate =~ s/\s*MAILED ON\s*//g;
   ($pubmonth, $pubday, $pubyear) = &getDateParts($pubdate);
   ($mailmonth, $mailday, $mailyear) = &getDateParts($maildate);
   
   # Get rid of useless data
   ($firsttag) = ($input =~ /.*?(<AWARD SEP|<AWARD>|<PRESOL|<MOD|<SRCGST|<SNOTE|<SSALE|<FSTD)/s);
   ($input) = ($input =~ /($firsttag.*)/s);
   while($input)
   {
      # Get the starting and ending tag of first chunk

      @req = ();
      @req = split(/\n/, $input);
      if($req[0] =~ /PRESOL/)
      {
         $endtag = "</PRESOL>";
         $type= "pre-solicit";
      }
      elsif($req[0] =~ /MOD/)
      {
         $endtag = "</MOD>";
         $type = "modification";
      }
      elsif($req[0] =~ /AWARD/)
      {
         $endtag = "</AWARD>";
         $type = "award";
      }
      elsif($req[0] =~ /SRCGST/)
      {
         $endtag = "</SRCGST>";
         $type = "source-sought";
      }
      elsif($req[0] =~ /FSTD/)
      {
         $endtag = "</FSTD>";
         $type = "foreign.standard";
      }
      elsif($req[0] =~ /SNOTE/)
      {
         $endtag = "</SNOTE>";
         $type = "special-note";
      }
      elsif($req[0] =~ /SSALE/)
      {
         $endtag = "</SSALE>";
         $type = "surplus.sale";
      }

      # get the actual chunk
      ($reqchunk, $input) = ($input =~ /(.*?$endtag)(.*)/s); 
      $reqchunk =~ s/\"/&quot;/gsi;

      # get the date from chunk
      ($date) = ($reqchunk =~ /(<DATE>.*?<\/DATE>)/s);
      $reqchunk =~ s/$date//g;
      ($month, $day) = ($date =~ /.*(\d\d)(\d\d).*/);
      $month = $Months[$month];
      ($year) = ($reqchunk =~ /(<YEAR>.*?<\/YEAR>)/s);
      $reqchunk =~ s/$year//g;
      $year =~ s/<\/YEAR>//s;
      $year =~ s/<YEAR>//s;
      if($year < 70) { $year = 2000+ $year; }
      elsif($year < 100) { $year = 1900 + $year; }
     
      # Additional cleanup
      $reqchunk =~ s/$firsttag.*?>//;
      $reqchunk =~ s/$endtag//; 

      # Spit out recd
      ($recd) = ($reqchunk =~ /(<RECD>.*?<\/RECD>)/s);
      ($recd) = ($recd =~ /<RECD>(.*)<\/RECD>/s);
      ($recd) = ($recd =~ /(\w-\d\d\d SN\d\d\d\d\d\d)/);
      $recd = &trim($recd);
      ($submitted, $submissionnumber) = ($recd =~ /(\S+)\s(.*)/);
      $origsub = $submitted;
      $origsub =~ s/\(//gsi;
      $origsub =~ s/\)//gsi;
      $origsub =~ s/-//gsi;
      $origsub =~ s/[a-zA-Z]//gsi;
      if($submitted =~ /I/)
      {
         $submitted = "provider-email";
      }
      elsif($submitted =~ /D/)
      {
         $submitted = "direct-email";
      }
      elsif($submitted =~ /W/ )
      {
         $submitted = "web";
      }
      elsif($submitted =~ /M/)
      {
         $submitted = "manuscript";
      }
      elsif($submitted =~ /A/)
      {
         $submitted = "ftp";
      }

      $submissionnumber =~ s/\)//g;

      # Fix ClassCod
      ($classcod) = ($reqchunk =~ /(<CLASSCOD>.*?<\/CLASSCOD>)/s);
      ($desc) = ($classcod =~ /<CLASSCOD>(.*)<\/CLASSCOD>/s);
      $desc = trim($desc);
      if($desc =~ /^(\S+) - (.*)$/)
      {
         ($code, $desc) = ($1, $2);
      }
      $newclasscod = "<class.code code=\"$code\" desc=\"$desc\"/>\n";
      if($code =~ /[A-Z]/)
      {
         $subpart = "services";
      }
      elsif($code =~ /[0-9]/)
      {
         $subpart = "supplies";
      }
      $recd =~ s/\(//g;
      $recd =~ s/\)//g;
 
      # start spitting out into xmlout
      $subnum = $submissionnumber;
      $subnum =~ s/SN//gsi;
      $XMLout = "";
      $XMLout .= "<cbd name=\"doc.cbd.$pubyear.$origsub.$subnum\" >\n\n";
      $XMLout .= "<cbd.props number=\"$recd\" type=\"$type\"  submitted=\"$submitted\" subpart=\"$subpart\"/>\n";

      $XMLout .= "<doc.props>\n";
      $XMLout .= "<doc.submission>\n";
      $XMLout .= "<submission.front>\n";

      # Spit out pubdate, maildate and issue
      $XMLout .= "<pub.date month=\"$pubmonth\" day=\"$pubday\" year=\"$pubyear\" day-name=\"$pubdayname\"/>\n";
      $XMLout .= "<mail.date month=\"$mailmonth\" day=\"$mailday\" year=\"$mailyear\"/>\n";
      $issue =~ s/Issue No\.//gsi;
      $issue = &trim($issue);
      $XMLout .= "<issue.number>$issue</issue.number>\n";

      $XMLout .= "</submission.front>\n";
      $XMLout .= "<submission.document>\n";

      # Spit out Date
      $XMLout .= "<item.date month=\"$month\" day=\"$day\" year=\"$year\"/>\n";

      # Spit out zip
      ($zip) = ($reqchunk =~ /(<ZIP>.*?<\/ZIP>)/);
      $zip =~ s/<ZIP>/<zip>/;
      $zip =~ s/<\/ZIP>/<\/zip>/;
      $XMLout .= "$zip\n";

      $XMLout .= "$newclasscod\n";

      # Fix Offadd
      ($offadd) = ($reqchunk =~ /(<OFFADD>.*?<\/OFFADD>)/s);
      ($newoffadd) = ($offadd =~ /<OFFADD>(.*)<\/OFFADD>/s);
      ($office, $street, $city, $region, $zip, $country) = &getAddressParts($newoffadd);
      $newoffadd = "<office.address>\n";
      $newoffadd .= "<postal>\n";
      $street = &trim($street);
      $newoffadd .= "<street>$street</street>\n";
      $city = &trim($city);
      $newoffadd .= "<city>$city</city>\n";
      $region = &trim($region);
      $newoffadd .= "<region>$region</region>\n";
      $zip = &trim($zip);
      $newoffadd .= "<code>$zip</code>\n";
      if($country =~ /\w+/)
      {
         $country = &trim($country);
         $newoffadd .= "<country>$country</country>\n";
      }
      $newoffadd .= "</postal>\n";
      $office = &trim($office);
      $newoffadd .= "<office>$office</office>\n";
      $newoffadd .= "</office.address>\n";
      if($offadd =~ /\w+/)
      {
         $XMLout .= "$newoffadd\n";
      }

      # Fix subject
      ($subject) = ($reqchunk =~ /(<SUBJECT>.*?<\/SUBJECT>)/s);
      ($desc) = ($subject =~ /<SUBJECT>(.*)<\/SUBJECT>/s);
      $desc = &trim($desc);
      if($desc =~ /^(\S+) - (.*)$/)
      {
         ($code, $desc) = ($1, $2);
      }
      $newsubject = "<subject code=\"$code\" desc=\"$desc\"/>\n";
      if($subject =~ /\w+/)
      {
         $XMLout .= "$newsubject\n";
      }

      # Spit out solnbr
      ($solnbr) = ($reqchunk =~ /(<SOLNBR>.*?<\/SOLNBR>)/s);
      $solnbr =~ s/<SOLNBR>/<solicit.number>/;
      $solnbr =~ s/<\/SOLNBR>/<\/solicit.number>/;
      $XMLout .= "$solnbr\n";
      
      # Fix respdate
      ($respdate) = ($reqchunk =~ /(<RESPDATE>.*?<\/RESPDATE>)/s);
      ($newrespdate) = ($respdate =~ /<RESPDATE>(.*)<\/RESPDATE>/s);
      $newrespdate =~ s/DUE\s*//g;
      ($month, $day, $year) = ($respdate =~ /(\d\d)(\d\d)(\d+)/);
      $month = $Months[$month];
      if($year < 70) { $year = 2000 + $year; }
      elsif($year > 70 && $year < 100) { $year = 1900 + $year; }
      $newrespdate = "<response.date month=\"$month\" day=\"$day\" year=\"$year\"/>\n";
      if($respdate =~ /\w+/)
      {
         $XMLout .= "$newrespdate\n";
      }

      # Fix contact
      ($contact) = ($reqchunk =~ /(<CONTACT>.*?<\/CONTACT>)/s);
      ($newcontact) = ($contact =~ /<CONTACT>(.*)<\/CONTACT>/s);
      # @phone = &getContactPhone($newcontact);
      # @mail = &getEmailAddress($newcontact);
      $newcontact =~ s/;/;POC /;
      $newcontact =~ s/ or / or POC/i;
      @contacts = split(/,| ; | or | at |\//, $newcontact);
      $newcontact = "<contact>\n"; 
      my(@splitlines);
      my($prev, $firstname, $middlename, $lastname, $s, $far, $url1, $url2);
      my($dfar, $sic, $awdnbr, $awdamt, $linenbr);
      my($outfilename, $dir);

      foreach my $u (@contacts)
      {
         if($u =~ /POC/i || $u =~ /Points* of contact/i || $u =~ /Contact Point/i) 
         { 
            if($prev =~ /\(?\d\d\d\)?\s*-*\d\d\d\s*-*\d\d\d\d/)
            {
               $newcontact .= "/>\n";
               $prev = "";
            }
            $u =~ s/POC//;
            $u =~ s/Points* of contact//gsi;
            $u =~ s/Contact Point//gsi;
            $u = &trim($u);
            ($firstname, $middlename, $lastname) = split(/\s/, $u);
            if($lastname =~ /\w+/)
            {
               # Do nothing
            }
            else
            {
               $lastname = $middlename;
               $middlename = "";
            }
            $newcontact .= "<person.name last-name=\"$lastname\" first-name=\"$firstname\" middle-name=\"$middlename\" full-name=\"$u\" />\n"; 
         }
         elsif($u =~ /\(?\d\d\d\)?\s*-*\d\d\d\s*-*\d\d\d\d/)
         {
            if($prev =~ /\(?\d\d\d\)?\s*-*\d\d\d\s*-*\d\d\d\d/) {
              $prev = "";
            }

            if($u =~ /Fax/si) {
              ($u) = ($u =~ /(\(?\d\d\d\)?\s*-*\d\d\d\s*-*\d\d\d\d)/);
              $u =~ s/Fax//si;
              $u = &trim($u);
              $newcontact .= "<facsimile>$u</facsimile>" . "\n";
            }
            else {
              ($u) = ($u =~ /(\(?\d\d\d\)?\s*-*\d\d\d\s*-*\d\d\d\d)/);
              $u =~ s/Phone//si;
              $u = &trim($u);
              $newcontact .= "<phone>$u</phone>" . "\n";
            }

            $prev = $u;
         }
         elsif($u =~ /\S+@\S+/)
         {
            if($prev =~ /\(?\d\d\d\)?\s*-*\d\d\d\s*-*\d\d\d\d/)
            {
               $newcontact .= "/>\n";
               $prev = "";
            } 
            $u =~ s/Email//i;
            ($u) = ($u =~ /(.*@.*\.\w\w\w?)/);
            $u =~ s/&quot;//gsi;
            $u = &trim($u);
            $newcontact .= "<cbd.email email=\"$u\"/>\n";
         }
         else
         {
            if($prev =~ /\(?\d\d\d\)?\s*-*\d\d\d\s*-*\d\d\d\d/)
            {
##               $newcontact .= "/>\n";
               $prev = "";
            } 
            $u = &trim($u);
            $newcontact .= "<person.title>$u</person.title>\n";
         }
         if($prev =~ /\(?\d\d\d\)?\s*-*\d\d\d\s*-*\d\d\d\d/)
         {
##            $newcontact .= "/>\n";
            $prev = "";
         }
      } 
      $newcontact .= "</contact>\n";
      if($contact =~ /\w+/)
      {
          $XMLout .= "$newcontact\n";
      }

      # Fix desc
      ($desc) = ($reqchunk =~ /(<DESC>.*?<\/DESC>)/s);
      ($desc) = ($desc =~ /<DESC>(.*)<\/DESC>/);

      # Fix split lines
      @splitlines = split(/\n/, $desc);
      $desc = "";
      foreach $s(@splitlines)
      {
         $desc .= $s;
      }
      $s =~ s/\!\!//gsi;

      $newdesc = "<desc>\n";

      # Skulked stuff

      $newdesc .= "<skulked.desc>\n";

      # Look for links
      $found = 1;
      $temp = $desc;
      while($found)
      {
         if($temp =~ /http:\/\/\S+/)
         {
            ($link, $temp) = ($temp =~ /(http:\/\/\S+)(.*)/s);
            $link =~ s/\(//;
            $link =~ s/\)//;
            $link =~ s/\*//g;
            $link =~ s/mdash;/_/gsi;
            $link =~ s/llmdash;/_/gsi;
            $link =~ s/&quot;//gsi;
            ($link) = ($link =~ /(.*\w)/);
            if($newdesc !~ /$link/)
            {
               $newdesc .= "<link link-type=\"info\" uri=\"$link\"/>\n";
               if($debug)
               {
                  print STDERR $link, "\n";
               }
            }
            $link = "";
         }
         else
         {
            $found = 0;
         }
      }

      # Look for email addresses
      $found = 1;
      $temp = $desc;
      while($found)
      {
         if($temp =~ /\S+@\S+/)
         {
            ($email, $temp) = ($temp =~ /(\S+@\S+)(.*)/);
            ($email) = ($email =~ /(.*@.*\.[a-zA-Z][a-zA-Z][a-z][A-Z]?)/);
            $email =~ s/\(//;
            $email =~ s/\)//;
            $email =~ s/\*//g;
            $email =~ s/\[//g;
            $email =~ s/\]//g;
            $email =~ s/&quot;//gsi;
            $email =~ s/.*://gsi;
             
            if($email =~ /@.*\.[a-zA-Z][a-zA-Z][a-zA-Z]?$/)
            {
            if($newdesc !~ /$email/ && $email =~ /\w+/)
            {
               $newdesc .= "<cbd.email email=\"$email\"/>\n";
               if($debug)
               {
                  print STDERR $email, "\n";
               }
            }
            $email = "";
            }
         }
         else
         {
            $found = 0;
         }
      } 

      # Look for FARs
      $found = 1;
      $temp = $desc;
      while($found)
      {
         if($temp =~ /FAR \d+\.\d+/)
         {
            ($far, $temp) = ($temp =~ /(FAR .*?\s)(.*)/);
            ($url1, $url2) = ($far =~ /(\d+)\.(\d+)/);
            $url2 = $url1. "." . $url2;
            if(length($url1) eq 1)
            {
               $url1 = "0".$url1;
            }
            if($far =~ /\d+/ && $newdesc !~ /$url1\.html#$url2/)
            {
               $newdesc .= "<link link-type=\"far\" uri=\"http://www.arnet.gov/far/current/html/$url1.html#$url2\" desc=\"$far\" />\n";
               if($debug)
               {
                  print STDERR $far, "\n";
               }
            }
            $far = "";
         }
         else
         {
            $found = 0;
         }
      }

      # Looking for DFARS
      $found = 1;
      $temp = $desc;
      while($found)
      {
         if($temp =~ /DFARS* \d+\.\d+/)
         {
            ($dfar, $temp) = ($temp =~ /(DFARS* .*?\s)(.*)/);
            ($url1, $url2) = ($dfar =~ /(\d+)\.(\d+)/);
            if($dfar =~ /\d+/)
            {
               $newdesc .= "<link link-type=\"dfars\" uri=\"http://www.acq.osd.mil/dp/dars/html/$url1";
               $newdesc .= "_";
               $newdesc .= "$url2.html\" desc=\"$dfar\" />\n";
            }
            $dfar = "";
         }
         else
         {
            $found = 0;
         }
      } 

      # Looking for SICs
      $found = 1;
      $temp = $desc;
      while($found)
      {
         if($temp =~ /SIC.*?\d\d\d\d/)
         {
            ($sic, $temp) = ($temp =~ /(SIC.*?\d\d\d\d)(.*)/s);
            ($sic) = ($sic =~ /.*(\d\d\d\d)/);
            if($debug)
            {
               print STDERR $sic, "\n";
            }
            $sic = "http://edgar.space.invisible.net/help/edgar/sic/sic_" . $sic . ".html";
            if($newdesc !~ /$sic/)
            {
               $newdesc .= "<link link-type=\"sic\" uri=\"$sic\"/>\n";
            }
         }
         else
         {
            $found = 0;
         }
      }
    
      $newdesc .= "<\/skulked.desc>\n";
      
      # Unskulked stuff
      $newdesc .= "<unskulked.desc>\n";
      $newdesc .= "$desc\n";
      $newdesc .= "</unskulked.desc>\n";
      $newdesc .= "</desc>\n";
    
      $XMLout .= "$newdesc\n";

      # Fix link
      ($link) = ($reqchunk =~ /(<LINK>.*?<\/LINK>)/s);
      ($uri) = ($link =~ /<URL>(.*)<\/URL>/s);
      ($desc) = ($link =~ /<DESC>(.*)<\/DESC>/s);
      $uri =~ s/\n/ /gsi;
      $desc =~ s/\n/ /gsi;
      $uri = &trim($uri);
      $uri =~ s/mdash;/_/gsi;
      $uri =~ s/llmdash;/_/gsi;
      $desc = &trim($desc);
      $newlink = "<link link-type=\"info\" uri=\"$uri\" desc=\"$desc\"/>\n";
      if($link =~ /\w+/)
      {
         $XMLout .= "$newlink\n";
      }

      # Fix email
      ($email) = ($reqchunk =~ /(<EMAIL>.*?<\/EMAIL>)/s);
      ($address) = ($email =~ /<ADDRESS>(.*)<\/ADDRESS>/s);
      ($desc) = ($email =~ /<DESC>(.*)<\/DESC>/s);
      $address =~ s/\n/ /gsi;
      $desc =~ s/\n/ /gsi;
      $address = &trim($address);
      $desc = &trim($desc);
      $newemail = "<cbd.email email=\"$address\" desc=\"$desc\"/>\n";
      if($email =~ /\w+/)
      {
         $XMLout .= "$newemail\n";
      }

      # Spit out awdnbr
      ($awdnbr) = ($reqchunk =~ /(<AWDNBR>.*?<\/AWDNBR>)/s);
      $awdnbr = &trim($awdnbr);
      $awdnbr =~ s/<AWDNBR>/<award.number>/;
      $awdnbr =~ s/<\/AWDNBR>/<\/award.number>/;
      $XMLout .= "$awdnbr\n"; 

      # Spit out awardamount
      ($awdamt) = ($reqchunk =~ /(<AWDAMT>.*?<\/AWDAMT>)/s);
      ($awdamt) = ($awdamt =~ /<AWDAMT>(.*)<\/AWDAMT>/s);
      $awdamt =~ s/AMT //gsi;
      $awdamt =~ s/\$//gsi;
      $awdamt =~ s/,//gsi; 
      $awdamt = &trim($awdamt);
      if($awdamt =~ /\d+/)
      {
         $awdamt = "<award.amount dollar-amount=\"$awdamt\" />\n"; 
         $XMLout .= "$awdamt\n";
      }

      # Spit out linenbr
      ($linenbr) = ($reqchunk =~ /(<LINENBR>.*?<\/LINENBR>)/s);
      $linenbr = &trim($linenbr);
      $linenbr =~ s/<LINENBR>/<line.number>/;
      $linenbr =~ s/<\/LINENBR>/<\/line.number>/;
      $XMLout .= "$linenbr\n";

      # Fix awarddate
      ($awarddate) = ($reqchunk =~ /(<AWDDATE>.*?<\/AWDDATE>)/s);
      ($newawarddate) = ($awarddate =~ /<AWDDATE>(.*)<\/AWDDATE>/s);
      ($month, $day, $year) = ($newawarddate =~ /(\d\d)(\d\d)(\d+)/);
      $month = $Months[$month];
      if($year < 70) { $year = 2000 + $year; }
      elsif($year > 70 && $year < 100) { $year = 1900 + $year; }
      $newawarddate = "<award.date month=\"$month\" day=\"$day\" year=\"$year\"/>\n";
      if($awarddate =~ /\w+/)
      {
         $XMLout .= "$newawarddate\n";
      }

      # Fix awardee
      ($awardee) = ($reqchunk =~ /(<AWARDEE>.*?<\/AWARDEE>)/s);
      ($newawardee) = ($awardee =~ /<AWARDEE>(.*)<\/AWARDEE>/s);
      ($office, $street, $city, $region, $zip, $country) = &getAddressParts($newawardee);
      $newawardee = "<awardee>\n";
      $newawardee .= "<postal>\n";
      $street = &trim($street);
      $newawardee .= "<street>$street</street>\n";
      $city = &trim($city);
      $newawardee .= "<city>$city</city>\n";
      $region = &trim($region);
      $newawardee .= "<region>$region</region>\n";
      $zip = &trim($zip);
      $newawardee .= "<code>$zip</code>\n";
      if($country =~ /\w+/)
      {
         $country = &trim($country);
         $newawardee .= "<country>$country</country>\n";
      }
      $newawardee .= "</postal>\n";
      $office = &trim($office);
      $newawardee .= "<office>$office</office>\n";
      $newawardee .= "</awardee>\n";
      if($awardee =~ /\w+/)
      {
         $XMLout .= "$newawardee\n";
      }

      $XMLout .= "</submission.document>\n</doc.submission>\n";
      $XMLout .= "</doc.props>\n";
      $XMLout .= "<remote.props uri=\"http://frwebgate.access.gpo.gov/cgi-bin/getdoc.cgi?dbname=HTMLCBD_ALL&amp;docid=cite=";
      $XMLout .= $submissionnumber;
      $XMLout .= "\" language=\"html\" />\n"; 
      $XMLout .= "</cbd>\n\n";

      # Stuff in cbd name
      
      # Perform encoding
## TODO: always use xml lib encoding routine
##      $XMLout =~ s/llmdash;//gsi; 
##      $XMLout =~ s/mdash;//gsi;
##      $XMLout =~ s/emsp;/ /gsi;
##      $XMLout =~ s/ensp;/ /gsi;
##      $XMLout =~ s/gt;/\&gt;/gsi;
##      $XMLout =~ s/lt;/\&lt;/gsi;
##      $XMLout =~ s/amp;/\&amp;/gsi;
##      $XMLout =~ s/ndash;//gsi;
##      $XMLout =~ s/&&/&/gsi;
##      $XMLout =~ s/&amp;quot;/&quot;/gsi;

## TODO: this should be a function
      $outfilename = $recd;
      $outfilename =~ s/\(//gsi;
      $outfilename =~ s/\)//gsi;
      $outfilename .= ".xml";
      $dir = "/data/cbd/xml/";
      $dir .= "$pubyear" . "/";

## TODO: just build the path then use makepath()
      if(!opendir(DIR,$dir))
      {  
         # Create the directory
         mkdir($dir, 0777);
      }
      else
      {
         closedir(DIR);
      }
      $dir =~ s/-//gsi;
      $dir =~ s/ /_/gsi;
      $dir .= "$pubmonth" . "/";
      if(!opendir(DIR,$dir))
      {
         # Create the directory
         mkdir($dir, 0777);
      }
      else
      {
         closedir(DIR);
      }
##      $dir =~ s/-//gsi;
##      $dir =~ s/ /_/gsi; 
##      ($file1) = ($file =~ /.*\/(.*)\.xml/);
##      $dir .= "$file1" . "/";
##      $dir =~ s/ //gsi;
##      $dir =~ s/-//gsi;
##      $dir =~ s/ /_/gsi;
##      if(!opendir(DIR,$dir))
##      {
##         # Create the directory
##         mkdir($dir, 0777);
##      }
##      else
##      {
##         closedir(DIR);
##      }
      $dir .= "$outfilename";
      $dir =~ s/-//gsi;
      $dir =~ s/ /_/gsi; 

      if($outfilename =~ /SN\d+/)
      {
         # output filename for live CBD processing
         print $dir, "\n";

         open (OUT, ">$dir") or warn "cant open file";
         print OUT $XMLout, "\n";
         close OUT;
      }
      else
      {
         print STDERR "Error! RECD tag is empty in $file\n";
      }
      # $XMLout = "<!DOCTYPE cbd SYSTEM \"/data/cbd/etc/cbdspace.dtd\">\n";
    
      # print STDERR $reqchunk, "\n";
      # print STDERR "-----------------------------------------------------------------\n";

      # get next start tag
      ($firsttag) = ($input =~ /.*?(<AWARD|<PRESOL|<MOD|<SRCGST|<SNOTE|<SSALE|<FSTD)/s);
      if($firsttag =~ /\w+/)
      {
         ($input) = ($input =~ /($firsttag.*)/s);
      }
      else
      {
         $input = "";
      } 
      # print STDERR "-----------------------------------------------------------------\n";
   }
}

=head1 NAME

cbd2dtd - CBD Skulker

=head1 PACKAGE

CBD::cbd2dtd

=head1 REQUIRES

Perl, version 5.001 or higher.

=head1 DESCRIPTION

CBD Skulking routines

=over 3

=item example

On the command line, type cbd2dtd 01NO99.xml. This will extract all the different CBD documents from the file, convert them into XML valid with the DTD and write them to disk.

=head1 METHODS

=head2 getDateParts

=item *

Accepts a date string and returns month, day and year

=item example:

($month, $day, $year) = &CBD::cbd2dtd::getDateParts($dateStr);

=head2 getAddressParts

=item *

Accepts a postal string and returns office, street, city, state, zip code and country.

=item example:

($office, $street, $city, $state, $zip, $country) = &CBD::cbd2dtd($addStr);

=back

=head1 COPYRIGHT

Copyright 1999 Invisible Worlds

=head1 AUTHOR

CBD::cbd2dtd was written by Gautam Yegnanarayan <gautam@invisible.net>.

=cut