Switch from long to size_t when allocating memory.

Also move the declarations of `incore` and `core_alloc` to "memory.h".
Also correct SYMDEBUG to SYMDBUG.  (I don't know if SYMDBUG works
because our build system never defines it.)

ind_t becomes an alias of size_t.  ind_t becomes unsigned, so I edit
some code that was using negative ind_t.  Some casts disappear, like
(long)sizeof(...) because the size is already a size_t.  There are
changes to overflow checks.  Callers with a size too big for size_t
must check it before calling the memory allocator.  An overflow check
of BASE + incr in memory.c sbreak() now happens on all platforms, not
only when a pointer is smaller than a long.

My build shows no changes in share/ack/examples (except hilo_bas.*
changing with every build).

util/led/archive.c

@@ -41,14 +41,18 @@ getsymdeftable()
 
 	count = nran = rd_long(infile);
 	debug("%ld ranlib structs, ", nran, 0, 0, 0);
-	off = hard_alloc(ALLORANL, nran * sizeof(struct ranlib));
+	if (nran > SIZE_MAX / sizeof(struct ranlib))
+		off = BADOFF;	/* nran * size would overflow. */
+	else
+		off = hard_alloc(ALLORANL, nran * sizeof(struct ranlib));
 	if (off == BADOFF)
 		fatal("no space for ranlib structs");
 	ran = (struct ranlib *)address(ALLORANL, off);
 	rd_ranlib(infile, ran, count);
 	nchar = rd_long(infile);
 	debug("%ld ranlib chars\n", nchar, 0, 0, 0);
-	if ((off = hard_alloc(ALLORANL, nchar)) == BADOFF)
+	if (nchar != (size_t)nchar ||
+	    (off = hard_alloc(ALLORANL, nchar)) == BADOFF)
 		fatal("no space for ranlib strings");
 	rd_bytes(infile, address(ALLORANL, off), nchar);
 	ran = (struct ranlib *)address(ALLORANL, (ind_t)0);
@@ -144,7 +148,7 @@ notelib(pos)
 {
 	register ind_t	off;
 
-	if ((off = hard_alloc(ALLOARCH, (long)sizeof(long))) == BADOFF)
+	if ((off = hard_alloc(ALLOARCH, sizeof(long))) == BADOFF)
 		fatal("no space for archive position");
 	*(long *)address(ALLOARCH, off) = pos;
 }

util/led/finish.c

@@ -17,7 +17,6 @@ static char rcsid[] = "$Id$";
 #include "orig.h"
 #include "scan.h"
 
-extern bool	incore;
 extern unsigned short	NLocals;
 extern int	flagword;
 extern struct outname	*searchname();
@@ -127,8 +126,6 @@ handle_relos(head, sects, names)
 	register int		sectindex;
 	register int		nrelo;
 	register char		*emit;
-	extern char		*getemit();
-	extern struct outrelo	*nextrelo();
 	static long zeros[MAXSECT];
 
 	if (incore) {
@@ -169,7 +166,6 @@ handle_relos(head, sects, names)
 				    long sz = sects[sectindex].os_flen;
 				    long sf = 0;
 				    long blksz;
-				    char *getblk();
 
 				    emit = getblk(sz, &blksz, sectindex);
 				    while (sz) {

util/led/main.c

@@ -23,7 +23,6 @@ static char rcsid[] = "$Id$";
 #include "orig.h"
 #include "sym.h"
 
-extern bool	incore;
 #ifndef NOSTATISTICS
 int		statistics;
 #endif

util/led/memory.c

@@ -50,14 +50,10 @@ static ind_t refused;
 
 static int sbreak(ind_t incr)
 {
-	unsigned int	inc;
-
 	incr = (incr + (GRANULE - 1)) & ~(GRANULE - 1);
 
-	inc = incr;
 	if ((refused && refused < incr) ||
-	    (sizeof(char *) < sizeof(long) &&
+	    BASE + incr < BASE ||
-	     (inc != incr || BASE + inc < BASE)) ||
 	    brk(BASE + incr) == -1) {
 		if (!refused || refused > incr)
 			refused = incr;
@@ -150,12 +146,10 @@ move_up(piece, incr)
 
 	debug("move_up(%d, %d)\n", piece, (int)incr, 0, 0);
 	while (incr > 0 && sbreak(incr) == -1)
-		incr -= INCRSIZE;
+		incr -= INCRSIZE < incr ? INCRSIZE : incr;
 
-	if (incr <= 0) {
+	if (incr == 0)
-		incr = 0;
 		return (ind_t) 0;
-	}
 #ifndef NOSTATISTICS
 	if (statistics) fprintf(stderr,"moving up %lx\n", (long) incr);
 #endif
@@ -358,14 +352,14 @@ static int alloctype = NORMAL;
  * how many times the area is moved, because of another allocate, this offset
  * remains valid.
  */
-ind_t alloc(int piece, long size)
+ind_t alloc(int piece, size_t size)
 {
 	register ind_t		incr = 0;
 	ind_t			left = mems[piece].mem_left;
 	register ind_t		full = mems[piece].mem_full;
 
 	assert(passnumber == FIRST || (!incore && piece == ALLOMODL));
-	if (size == (long)0)
+	if (size == 0)
 		return full;
 	if (size != (ind_t)size)
 		return BADOFF;
@@ -373,13 +367,18 @@ ind_t alloc(int piece, long size)
 	case ALLOMODL:
 	case ALLORANL:
 		size = int_align(size);
+		if (size == 0)
+			return BADOFF;
 	}
 
-	if (size - left > 0)
+	if (size > left) {
 		incr = ((size - left + (INCRSIZE - 1)) / INCRSIZE) * INCRSIZE;
+		if (incr == 0)
+			return BADOFF;
+	}
 
 	if (incr == 0 ||
-	    (incr < left + full && (incr -= move_up(piece, left + full)) <= 0) ||
+	    (incr < left + full && move_up(piece, left + full) >= incr) ||
 	    move_up(piece, incr) == incr ||
 	    compact(piece, size, alloctype)) {
 		mems[piece].mem_full += size;
@@ -396,7 +395,7 @@ ind_t alloc(int piece, long size)
  * attempt fails, release the space occupied by other pieces and try again.
  */
 ind_t
-hard_alloc(int piece, long size)
+hard_alloc(int piece, size_t size)
 {
 	register ind_t	ret;
 	register int	i;
@@ -477,9 +476,7 @@ dealloc(int piece)
 }
 
 char *
-core_alloc(piece, size)
+core_alloc(int piece, size_t size)
-	register int	piece;
-	register long	size;
 {
 	register ind_t	off;
 
@@ -493,16 +490,8 @@ void core_free(int piece, char* p)
 	char	*q = address(piece, mems[piece].mem_full);
 
 	assert(p < q);
-	switch(sizeof(unsigned) == sizeof(char *)) {
+	mems[piece].mem_full -= (ind_t) q - (ind_t) p;
-	case 1:
+	mems[piece].mem_left += (ind_t) q - (ind_t) p;
-		mems[piece].mem_full -= (unsigned) (q - p);
-		mems[piece].mem_left += (unsigned) (q - p);
-		break;
-	default:
-		mems[piece].mem_full -= (ind_t) q - (ind_t) p;
-		mems[piece].mem_left += (ind_t) q - (ind_t) p;
-		break;
-	}
 }
 
 /*

util/led/memory.h

@@ -4,18 +4,21 @@
  */
 /* $Id$ */
 
+#include <stdbool.h>
+#include <stddef.h>
+
 #define ALLOEMIT	0			/* Section contents. */
 #define ALLORELO	(ALLOEMIT + MAXSECT)	/* Relocation table. */
 #define ALLOLOCL	(ALLORELO + 1)		/* Saved local names. */
 #define ALLOGLOB	(ALLOLOCL + 1)		/* Saved global names. */
 #define ALLOLCHR	(ALLOGLOB + 1)		/* Strings of local names. */
 #define ALLOGCHR	(ALLOLCHR + 1)		/* Strings of global names. */
-#ifdef SYMDEBUG
+#ifdef SYMDBUG
 #define ALLODBUG	(ALLOGCHR + 1)		/* Symbolic debugging info. */
-#else /* SYMDEBUG */
+#define ALLOSYMB	(ALLODBUG + 1)
-#define ALLODBUG	ALLOGCHR
+#else /* SYMDBUG */
-#endif /* SYMDEBUG */
+#define ALLOSYMB	(ALLOGCHR + 1)		/* Symbol table. */
-#define ALLOSYMB	(ALLODBUG + 1)		/* Symbol table. */
+#endif /* SYMDBUG */
 #define ALLOARCH	(ALLOSYMB + 1)		/* Archive positions. */
 #define ALLOMODL	(ALLOARCH + 1)		/* Modules. */
 #define ALLORANL	(ALLOMODL + 1)		/* Ranlib information. */
@@ -23,7 +26,7 @@
 
 #define BADOFF		((ind_t)-1)
 
-typedef long		ind_t;
+typedef size_t		ind_t;
 
 struct memory {
 	char	*mem_base;
@@ -35,13 +38,15 @@ extern struct memory	mems[];
 #define address(piece,offset)	(mems[(piece)].mem_base+(offset))
 #define modulptr(offset)	(mems[ALLOMODL].mem_base+core_position+(offset))
 
-#define int_align(sz)		(((sz)+(sizeof(int)-1))&~(int)(sizeof(int)-1))
+#define int_align(sz)		(((sz)+(sizeof(int)-1))&~(sizeof(int)-1))
 
+extern bool		incore;
 extern ind_t		core_position;
 extern void         init_core(void);
-extern ind_t        hard_alloc(int piece, long size);
+extern ind_t        hard_alloc(int piece, size_t size);
-extern ind_t        alloc(int piece, long size);
+extern ind_t        alloc(int piece, size_t size);
 extern void         dealloc(int piece);
+extern char         *core_alloc(int piece, size_t size);
 extern void         core_free(int piece, char* p);
 extern void         write_bytes(void);
 extern void         namecpy(struct outname* name, unsigned nname, long offchar);

util/led/memory_layout

@@ -21,10 +21,10 @@
 		-----------------------------------------------
 			Strings of global names *
 		-----------------------------------------------
-#ifdef SYMDEBUG
+#ifdef SYMDBUG
 			Symbolic debugging information
 		-----------------------------------------------
-#endif /* SYMDEBUG */
+#endif /* SYMDBUG */
 			Symbol table *
 		-----------------------------------------------
 			Archive positions *

util/led/output.c

@@ -18,7 +18,6 @@ static char rcsid[] = "$Id$";
 static void generate_section_names();
 
 extern struct outhead	outhead;
-extern bool		incore;
 extern int		flagword;
 
 /*
@@ -60,11 +59,10 @@ generate_section_names()
 {
 	register struct outname	*name;
 	register int		sectindex;
-	register long		size;
+	register size_t		size;
 	extern struct outsect	outsect[];
-	extern char		*core_alloc();
 
-	size = (long)outhead.oh_nsect * sizeof(struct outname);
+	size = outhead.oh_nsect * sizeof(struct outname);
 	name = (struct outname *)core_alloc(ALLOGLOB, size);
 	if (name == (struct outname *)0)
 		return;

util/led/save.c

@@ -21,9 +21,6 @@ static char rcsid[] = "$Id$";
 #include "const.h"
 #include "memory.h"
 
-extern bool	incore;
-extern char	*core_alloc();
-
 void
 savemagic()
 {
@@ -32,7 +29,7 @@ savemagic()
 	if (!incore)
 		return;
 
-	if ((p = core_alloc(ALLOMODL, (long)sizeof(int))) != (char *)0) {
+	if ((p = core_alloc(ALLOMODL, sizeof(int))) != (char *)0) {
 		*(unsigned short *)p = AALMAG;
 		core_position += sizeof(int);
 	}
@@ -47,7 +44,7 @@ savehdr(hdr)
 	if (!incore)
 		return;
 
-	if ((p=core_alloc(ALLOMODL,(long)sizeof(struct ar_hdr)))!=(char *)0) {
+	if ((p=core_alloc(ALLOMODL, sizeof(struct ar_hdr)))!=(char *)0) {
 		*(struct ar_hdr *)p = *hdr;
 		core_position += int_align(sizeof(struct ar_hdr));
 	}
@@ -66,7 +63,7 @@ savechar(piece, off)
 	register int	piece;
 	register ind_t	off;
 {
-	register long	len;
+	register size_t	len;
 	register ind_t	newoff;
 
 	if (off == (ind_t)0)
@@ -104,7 +101,7 @@ savelocal(name)
 		return;
 
 	new = (struct outname *)
-			core_alloc(ALLOLOCL, (long)sizeof(struct outname));
+			core_alloc(ALLOLOCL, sizeof(struct outname));
 	if (new != (struct outname *)0) {
 		*new = *name;
 		new->on_foff = savindex;

util/led/scan.c

@@ -13,11 +13,11 @@ static char rcsid[] = "$Id$";
 #include <stdbool.h>
 #include <string.h>
 #include <sys/types.h>
+#ifdef SYMDBUG
 #include <sys/stat.h>
+#endif /* SYMDBUG */
 #include <fcntl.h>
 #include <unistd.h>
-#ifdef SYMDBUG
-#endif /* SYMDBUG */
 #include "arch.h"
 #include "out.h"
 #include "ranlib.h"
@@ -35,8 +35,6 @@ static char rcsid[] = "$Id$";
 #define IND_DBUG(x)	(IND_RELO(x) + sizeof(ind_t))
 #endif /* SYMDBUG */
 
-extern char 	*core_alloc();
-extern bool	incore;
 extern int	infile;
 extern int	passnumber;
 
@@ -46,17 +44,17 @@ char		*modulname;	/* Name of object module. */
 long		objectsize;
 #endif /* SYMDBUG */
 
-static long	align(long size);
+static size_t	align(size_t size);
 static char	*modulbase;
-static long	modulsize(struct outhead* head);
+static size_t	modulsize(struct outhead* head);
 static void can_modul(void);
 static bool	all_alloc(void);
 static bool	direct_alloc(struct outhead* head);
 static bool	indirect_alloc(struct outhead* head);
 static bool putemitindex(ind_t sectindex, ind_t emitoff, int allopiece);
-static bool	putreloindex(ind_t relooff, long nrelobytes);
+static bool	putreloindex(ind_t relooff, size_t nrelobytes);
 #ifdef SYMDBUG
-static bool	putdbugindex(ind_t dbugoff, long ndbugbytes);
+static bool	putdbugindex(ind_t dbugoff, size_t ndbugbytes);
 #endif /* SYMDBUG */
 static void get_indirect(struct outhead* head, struct outsect* sect);
 static void read_modul(void);
@@ -77,7 +75,6 @@ getfile(filename)
 	unsigned short	magic_number;
 #ifdef SYMDBUG
 	struct stat	statbuf;
-	extern int	fstat();
 #endif /* SYMDBUG */
 
 	archname = (char *)0;
@@ -196,7 +193,7 @@ all_alloc(void)
 {
 	struct outhead	head;
 
-	if (hard_alloc(ALLOMODL, (long)sizeof(struct outhead)) == BADOFF)
+	if (hard_alloc(ALLOMODL, sizeof(struct outhead)) == BADOFF)
 		fatal("no space for module header");
 	rd_ohead((struct outhead *)modulptr(IND_HEAD));
 	/*
@@ -218,7 +215,7 @@ direct_alloc(head)
 	ind_t		sectindex = IND_SECT(*head);
 	register struct outsect *sects;
 	unsigned short	nsect = head->oh_nsect;
-	long		size, rest;
+	size_t		size, rest;
 
 #ifdef SYMDBUG
 	rest = nsect * sizeof(ind_t) + sizeof(ind_t) + sizeof(ind_t);
@@ -260,8 +257,7 @@ indirect_alloc(head)
 	ind_t		relooff = IND_RELO(*head);
 #ifdef SYMDBUG
 	ind_t		dbugoff = IND_DBUG(*head);
-	extern long	objectsize;
+	size_t		dbugsize = objectsize - OFF_DBUG(*head);
-	long		dbugsize = objectsize - OFF_DBUG(*head);
 #endif /* SYMDBUG */
 
 	assert(incore);
@@ -271,12 +267,14 @@ indirect_alloc(head)
 		sectindex += sizeof(struct outsect);
 		emitoff += sizeof(ind_t);
 	}
+	if (nrelo > SIZE_MAX / sizeof(struct outrelo))
+		return FALSE;	/* nrelo * size would overflow */
 #ifdef SYMDBUG
-	return	putreloindex(relooff, (long)nrelo * sizeof(struct outrelo))
+	return	putreloindex(relooff, nrelo * sizeof(struct outrelo))
 		&&
 		putdbugindex(dbugoff, dbugsize);
 #else /* SYMDBUG */
-	return putreloindex(relooff, (long)nrelo * sizeof(struct outrelo));
+	return putreloindex(relooff, nrelo * sizeof(struct outrelo));
 #endif /* SYMDBUG */
 }
 
@@ -302,6 +300,8 @@ putemitindex(ind_t sectindex, ind_t emitoff, int allopiece)
 
 	flen = ((struct outsect *)modulptr(sectindex))->os_flen;
 	if (flen && zero) {
+		if (zero != (size_t)zero)
+			return FALSE;
 		if ((emitindex = alloc(allopiece, zero)) != BADOFF){
 			register char *p = address(allopiece, emitindex);
 
@@ -313,6 +313,8 @@ putemitindex(ind_t sectindex, ind_t emitoff, int allopiece)
 	}
 	zeros[allopiece - ALLOEMIT] =
 	 zero + ((struct outsect *) modulptr(sectindex))->os_size - flen;
+	if (flen != (size_t)flen)
+		return FALSE;
 	if ((emitindex = alloc(allopiece, flen)) != BADOFF) {
 		*(ind_t *)modulptr(emitoff) = emitindex;
 		return TRUE;
@@ -325,7 +327,7 @@ putemitindex(ind_t sectindex, ind_t emitoff, int allopiece)
  * offset at `relooff'.
  */
 static bool
-putreloindex(ind_t relooff, long nrelobytes)
+putreloindex(ind_t relooff, size_t nrelobytes)
 {
 	ind_t		reloindex;
 
@@ -340,7 +342,7 @@ putreloindex(ind_t relooff, long nrelobytes)
  * Allocate space for debugging information and put the offset at `dbugoff'.
  */
 static bool
-putdbugindex(ind_t dbugoff, long ndbugbytes)
+putdbugindex(ind_t dbugoff, size_t ndbugbytes)
 {
 	ind_t		dbugindex;
 
@@ -417,12 +419,12 @@ read_modul(void)
 	char		*chars;
 	ind_t		sectindex, nameindex, charindex;
 	unsigned short	nsect, nname;
-	long		size;
+	size_t		size;
 	long		nchar;
 
 	assert(passnumber == SECOND);
 	assert(!incore);
-	if (hard_alloc(ALLOMODL, (long)sizeof(struct outhead)) == BADOFF)
+	if (hard_alloc(ALLOMODL, sizeof(struct outhead)) == BADOFF)
 		fatal("no space for module header");
 	head = (struct outhead *)modulptr(IND_HEAD);
 	rd_ohead(head);
@@ -457,11 +459,10 @@ read_modul(void)
  * Align `size' to a multiple of the size of a double.
  * This is assumed to be a power of 2.
  */
-static long
+static size_t
-align(size)
+align(size_t size)
-	register long	size;
 {
-	return (size + (sizeof(double) - 1)) & ~(int)(sizeof(double) - 1);
+	return (size + (sizeof(double) - 1)) & ~(sizeof(double) - 1);
 }
 
 /*
@@ -477,9 +478,8 @@ align(size)
  *	6. the offset of the debugging information.
 #endif
  */
-static long
+static size_t
-modulsize(head)
+modulsize(struct outhead *head)
-	register struct outhead	*head;
 {
 	return	sizeof(struct outhead) +			/* 0 */
 		head->oh_nsect * sizeof(struct outsect) +	/* 1 */
@@ -552,10 +552,13 @@ getemit(head, sects, sectindex)
 {
 	char		*ret;
 	ind_t		off;
-	extern char	*core_alloc();
+	long		flen;
 
 	if (!incore) {
-		ret = core_alloc(ALLOMODL, sects[sectindex].os_flen);
+		flen = sects[sectindex].os_flen;
+		if (flen != (size_t)flen)
+			return 0;
+		ret = core_alloc(ALLOMODL, flen);
 		if (ret == (char *)0)
 			return 0;
 		rd_outsect(sectindex);
@@ -581,6 +584,7 @@ getblk(totalsz, pblksz, sectindex)
 
 	assert(!incore);
 
+	while (sz != (size_t)sz) sz >>= 1;
 	while (sz >= totalsz) sz >>= 1;
 	while (sz) {
 		ret = core_alloc(ALLOMODL, sz);

util/led/scan.h

@@ -23,4 +23,6 @@ extern void get_modul(void);
 extern void skip_modul(struct outhead* head);
 extern void startrelo(struct outhead* head);
 extern struct outrelo* nextrelo(void);
+extern char* getemit(struct outhead* head, struct outsect* sects, int sectindex);
+extern char* getblk(long totalsz, long* pblksz, int sectindex);
 extern void endemit(char* emit);

util/led/sym.c

@@ -105,9 +105,9 @@ void entername(struct outname* name, int hashval)
 
 	debug("entername %s %d %x %x", modulptr((ind_t)name->on_foff), hashval, name->on_type, name->on_desc);
 	savindex = savechar(ALLOGCHR, (ind_t)name->on_foff);
-	symindex = hard_alloc(ALLOSYMB, (long)sizeof(struct symbol));
+	symindex = hard_alloc(ALLOSYMB, sizeof(struct symbol));
 	debug("; %ld\n", symindex, 0, 0, 0);
-	namindex = hard_alloc(ALLOGLOB, (long)sizeof(struct outname));
+	namindex = hard_alloc(ALLOGLOB, sizeof(struct outname));
 	if (savindex == BADOFF || symindex == BADOFF || namindex == BADOFF)
 		fatal("symbol table overflow");
 	sym = (struct symbol *)address(ALLOSYMB, symindex);

util/led/write.c

@@ -20,7 +20,6 @@ static char rcsid[] = "$Id$";
 extern struct outhead	outhead;
 extern struct outsect	outsect[];
 extern int		flagword;
-extern bool		incore;
 
 wr_fatal()
 {