head    1.12;
access;
symbols;
locks; strict;
comment @ * @;


1.12
date    94.03.12.00.59.25;      author paul;    state Exp;
branches;
next    1.11;

1.11
date    94.03.11.21.26.14;      author paul;    state Exp;
branches;
next    1.10;

1.10
date    94.01.07.18.04.25;      author paul;    state Exp;
branches;
next    1.9;

1.9
date    93.11.24.22.40.56;      author paul;    state Exp;
branches;
next    1.8;

1.8
date    93.04.16.02.11.11;      author paul;    state Exp;
branches;
next    1.7;

1.7
date    93.04.02.16.14.43;      author paul;    state Exp;
branches;
next    1.6;

1.6
date    93.04.01.16.27.58;      author paul;    state Exp;
branches;
next    1.5;

1.5
date    92.07.29.04.15.49;      author paul;    state Exp;
branches;
next    1.4;

1.4
date    92.07.28.16.14.12;      author paul;    state Exp;
branches;
next    1.3;

1.3
date    92.07.28.05.04.09;      author paul;    state Exp;
branches;
next    1.2;

1.2
date    92.07.27.22.20.02;      author paul;    state Exp;
branches;
next    1.1;

1.1
date    92.07.27.21.37.00;      author paul;    state Exp;
branches;
next    ;


desc
@@


1.12
log
@New copyright statement.
@
text
@/*
* Copyright (c) 1985 Corporation for Research and Educational Networking
* Copyright (c) 1988 University of Illinois Board of Trustees, Steven
*              Dorner, and Paul Pomes
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
*    notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
*    notice, this list of conditions and the following disclaimer in the
*    documentation and/or other materials provided with the distribution.
* 3. All advertising materials mentioning features or use of this software
*    must display the following acknowledgement:
*      This product includes software developed by the Corporation for
*      Research and Educational Networking (CREN), the University of
*      Illinois at Urbana, and their contributors.
* 4. Neither the name of CREN, the University nor the names of their
*    contributors may be used to endorse or promote products derived from
*    this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE TRUSTEES AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED.  IN NO EVENT SHALL THE TRUSTEES OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/

#ifndef lint
static char  RcsId[] = "@@(#)$Id$";
#endif

#include "protos.h"

/*
* these thing keep ld happy
*/
int     InputType;
int     Daemon;
FILE    *Input, *Output;
char    *DBState;

/*
* end of ld pacification
*/
extern int Quiet;               /* qi/qi.c */
extern int ReadOnly;
extern int DoTree;
int     CheckMeta;
static char *Me;                /* the name of this program */
extern int DoTree;
static int pipe_fd1[2], pipe_fd2[2];
static FILE *to_sort, *from_sort;
static void flush_key __P((char *,PTRTYPE *,PTRTYPE));
static int debug = 0;
extern void printarry __P((long *));
extern int getient __P((register char *, struct iindex *));

main(argc, argv)
       int     argc;
       char   **argv;
{
       char inbuf[MAX_KEY_LEN+50];
       char curkey[MAX_KEY_LEN+1] = "";
       PTRTYPE curmaxlen = NIPTRS, recidx = 0;
       PTRTYPE *reclist = (PTRTYPE *) calloc(curmaxlen,PTRSIZE);
       int count = 0, keys = 0;
       /* when you're strange, no one remembers your name */
       Me = *argv;

       OP_VALUE(NOLOG_OP) = strdup("");
       ReadOnly = 0;
       DoTree = 0;
       while (--argc > 0 && **(++argv) == '-')
         {
               char *equal, **opt;

               (*argv)++;
               if (**argv == 'q')
                       Quiet++;
               else if (equal = (char *)strchr(*argv, '='))
               {
                       *equal++ = 0;
                       for (opt = Strings; *opt; opt += 2)
                               if (!strcmp(opt[0], *argv))
                               {
                                       opt[1] = equal;
                                       break;
                               }
                       if (*opt == '\0')
                       {
                               fprintf(stderr, "%s: %s: unknown string.\n",
                                       Me, *argv);
                               exit(1);
                       }
               } else
               {
                       fprintf(stderr, "%s: %s: unknown option.\n", Me, *argv);
                       exit(1);
               }
       }
       Database = (argc > 0) ? *argv : DATABASE;
       if (!Quiet)
               printf("%s: indexing database %s\n", Me, Database);
       sleep(5);
       setbuf(stdout, NULL);

       DoSysLog(0);            /* report errors to stderr */

       dbd_init(Database);
       get_dir_head();
       if (!GetFieldConfig())
               exit(1);
       /* bintree_init(Database); *//* forget bintree here */
       DoTree = 0;
       print_head();

       /* set up the neccessary mechanism to talk to sort */
       pipe(pipe_fd1);
       pipe(pipe_fd2);

       if (fork() == 0)
         {
                 dup2(pipe_fd1[0], 0);
                 dup2(pipe_fd2[1], 1);

                 close(pipe_fd1[1]);
                 close(pipe_fd2[0]);

                 execlp("sort", "sort", "-t\t", "+0", "-1", "+1n", 0);
                 perror("Execl in makei");
                 exit(1);
         }
       close(pipe_fd1[0]);
       close(pipe_fd2[1]);

       to_sort = fdopen(pipe_fd1[1], "w");
       from_sort = fdopen(pipe_fd2[0], "r");


       if (fork() == 0)
         {             /* read from .dir file and write to sort */
               fclose(from_sort);
               printf("sent indicies for %d dir entries to sort.\n", make_index());
               exit(0);
         } else
           {           /* read from sort and insert into .idx file */
                   fclose(to_sort);

                   if (!dbi_init(Database)) {
                           fprintf(stderr,"%s: couldn't init\n",Database);
                           exit(1);
                   }
                   while (fgets(inbuf,sizeof(inbuf),from_sort)) {
                           char *rp = (char *)strchr(inbuf,'\t');
                           *rp++ = '\0';
                           count++;
                           if (!Quiet && count % 1000 == 1)
                             printf("%d from sort.\n", count);
                           if (strcmp(inbuf,curkey)) { /* new key */
                                   flush_key(curkey,reclist,recidx); /* flush the old one */
                                   recidx = 0;
                                   reclist[0] = 0;
                                   strncpy(curkey,inbuf,sizeof(curkey));
                                   keys++;
                           }
                           if (recidx >= curmaxlen-1)
                             reclist = (PTRTYPE *) realloc(reclist, (curmaxlen += NOPTRS)*PTRSIZE);
                           reclist[recidx++] = atoi(rp);
                   }
                   flush_key(curkey,reclist,recidx); /* flush the final one */
                   if (!Quiet)
                     printf("%d from sort\n", count);
           }
       printf("indexed %d unique strings out of %d total.\n",keys,count);

       exit(0);
}

extern int TrySum;
extern int TryTimes;
extern int WordsIndexed;
extern int MaxIdx;

make_index()
{
       QDIR    dirp;
       long    ent;
       extern struct dirhead DirHead;
       int     entries_done;
       void sort_lookup();


       entries_done = 0;

       for (ent = 1; ent < DirHead.nents; ent++)
       {
               if (!next_ent(ent))
               {
                       /* printf("didn't do %d\n",ent); */
                       continue;
               }
               getdata(&dirp); /* setup entry */

               /* for all make the index entries */

               MakeLookup(dirp, ent, sort_lookup);
               if ((entries_done++ % 1000) == 0)
                       printf("%d to sort\n", entries_done);
               FreeDir(&dirp);
       }
       return (ent);
}

print_head()
{
       extern struct dirhead DirHead;

       printf("nents = %d\n", DirHead.nents);
       printf("next_id = %d\n", DirHead.next_id);
}

void
sort_lookup(str,ent)
char *str;
int ent;
{
 char  buf[MAX_LEN];
 char  *cp;
 char *strlncpy();
#ifdef DEBUG
 static FILE *out = NULL;
 if (out == NULL) {
   if ((out = fopen("./debug.index","w")) == NULL) {
     perror("./debug.index");
     exit(1);
   }
 }
#endif
 (void) strlncpy(buf, str, MAX_LEN);
 for (cp = strtok(buf, IDX_DELIM); cp; cp = strtok(0, IDX_DELIM)) {
   if (cp[1] != '\0') {        /* has to be at least 2 letters! */
     fprintf(to_sort,"%s\t%d\n",cp,ent);
#ifdef DEBUG
     fprintf(out,"%s\t%d\n",cp,ent);
#endif
   }
 }
}

char *
strlncpy(to, from, max)
char *to, *from;
int max;
{
 char *save = to;

 while ((max-- > 0) && (*to++ = isupper(*from) ? tolower(*from) : *from))
   from++;

 return (save);
}

/*
* Flush the key and the entire list of records it is in.
*/

static void
flush_key(key,reclist,nelem)
char *key;
PTRTYPE *reclist, nelem;
{
 struct iindex x;
 long i, j, iloc;

 if (!*key)
   return;
 reclist[nelem] = 0;           /* make sure it's zero-terminated */
 key[MAX_KEY_LEN] = '\0';      /* just in case it isn't */

 if (!putstrarry(key, reclist)) {
   fprintf(stderr,"putstrarry failed for key %s (%d elements)\n",key,nelem);
   return;
 }
}
@


1.11
log
@Replace DontLog with Option setting macro.
@
text
@a0 2
#include "protos.h"

d2 33
a34 5
* This software is Copyright (C) 1988 by Steven Dorner and the
* University of Illinois Board of Trustees, and by CSNET.  No warranties of
* any kind are expressed or implied.  No support will be provided.
* This software may not be redistributed without prior consent of CSNET.
* You may direct questions to nameserv@@uiuc.edu
d36 6
@


1.10
log
@Replaced index() with strchr().
@
text
@a21 1
extern int DontLog;
d47 1
a47 1
       DontLog = 0;
@


1.9
log
@Improvements from Alan Crosswell <alan@@curta.cc.columbia.edu>.
@
text
@d58 1
a58 1
               else if (equal = index(*argv, '='))
d132 1
a132 1
                           char *rp = index(inbuf,'\t');
@


1.8
log
@Quiet now a global in qi/qi.c .
@
text
@d25 1
d29 6
d40 5
d48 1
a48 1
       DontLog = 1;
d50 1
d52 1
a52 1
       {
d81 1
a81 1
               fprintf(stderr, "%s: indexing database %s\n", Me, Database);
a86 1
       dbi_init(Database);
d94 60
a153 1
       printf("made indicies for %d dir entries\n", make_index());
d168 2
a169 1
       long    start_time, end_time;
a170 1
       (void) time(&start_time);
a182 1
               MakeLookup(dirp, ent, make_lookup);
d184 3
a186 9
               if ((entries_done++ % 100) == 0)
               {
                       (void) time(&end_time);
                       printf("%d %ld %d %d %d %d\n", entries_done,
                              end_time - start_time,
                              TrySum, TryTimes, WordsIndexed, MaxIdx);
                       MaxIdx = TrySum = TryTimes = 0;
                       start_time = end_time;
               }
d198 64
@


1.7
log
@Announce which database is being smashed, uhhhh built, and pause 5 seconds.
@
text
@d23 1
a32 2
       int Quiet = 0;

@


1.6
log
@Now can modify Strings[] like qi, e.g. prog -DATABASE=/tmp/foo .
@
text
@d68 3
@


1.5
log
@Deleted extraneous #include's.
@
text
@d25 1
d32 5
d39 1
a39 1
       if (argc != 2)
d41 25
a65 2
               printf("usage: %s database.\n", argv[0]);
               exit(1);
d67 1
d72 2
a73 2
       dbi_init(argv[1]);
       dbd_init(argv[1]);
d77 1
a77 1
       /* bintree_init(argv[1]); *//* forget bintree here */
@


1.4
log
@*** empty log message ***
@
text
@d1 2
a9 7

#include <ctype.h>
#include <stdio.h>
#include "protos.h"
#include "qi.h"
#include "db.h"
#include "field.h"
@


1.3
log
@Random fixes.
@
text
@d11 1
a14 1
#include "protos.h"
@


1.2
log
@Re-formatted for clarity
@
text
@d14 1
d66 1
a66 1
       DIR     dirp;
@


1.1
log
@Initial revision
@
text
@d1 7
a7 7
/*********************************************************************
* This software is Copyright (C) 1988 by Steven Dorner and the
* University of Illinois Board of Trustees, and by CSNET.  No warranties of
* any kind are expressed or implied.  No support will be provided.
* This software may not be redistributed without prior consent of CSNET.
* You may direct questions to dorner@@garcon.cso.uiuc.edu.
**********************************************************************/
d15 11
a25 11
/***********************************************************************
* these thing keep ld happy
***********************************************************************/
int   InputType;
int   Daemon;
FILE   *Input, *Output;
char *DBState;

/***********************************************************************
* end of ld pacification
***********************************************************************/
d28 1
a28 1
int   CheckMeta;
d32 2
a33 2
int   argc;
char  **argv;
d35 21
a55 20
 DontLog = 1;
 ReadOnly = 0;
 if (argc != 2)
 {
   printf("usage: %s database.\n", argv[0]);
   exit(1);
 }
 setbuf(stdout, NULL);

 DoSysLog(0);          /* report errors to stderr */

 dbi_init(argv[1]);
 dbd_init(argv[1]);
 get_dir_head();
 if (!GetFieldConfig()) exit(1);
 /* bintree_init(argv[1]); *//* forget bintree here */
 DoTree = 0;
 print_head();
 printf("made indicies for %d dir entries\n", make_index());
 exit(0);
d65 33
a97 34
 Dir   dir;
 long   ent;
 extern struct dirhead DirHead;
 int   entries_done;
 long  start_time, end_time;
 int   make_lookup();

 (void) time(&start_time);
 entries_done = 0;

 for (ent = 1; ent < DirHead.nents; ent++)
 {
   if (!next_ent(ent))
   {
     /* printf("didn't do %d\n",ent); */
     continue;
   }
   getdata(&dir);      /* setup entry */

   /* for all make the index entries */
   MakeLookup(dir, ent, make_lookup);

   if ((entries_done++ % 100) == 0)
   {
     (void) time(&end_time);
     printf("%d %ld %d %d %d %d\n", entries_done,
          end_time - start_time,
          TrySum, TryTimes, WordsIndexed, MaxIdx);
     MaxIdx = TrySum = TryTimes = 0;
     start_time = end_time;
   }
   FreeDir(&dir);
 }
 return (ent);
d102 1
a102 1
 extern struct dirhead DirHead;
d104 2
a105 2
 printf("nents = %d\n", DirHead.nents);
 printf("next_id = %d\n", DirHead.next_id);
@

You are viewing proxied material from mozz.us. The copyright of proxied material belongs to its original authors. Any comments or complaints in relation to proxied material should be directed to the original authors of the content concerned. Please see the disclaimer for more details.