Merge branch 'master' into next

* master:
  Generate defs in smaller chunks
  Generate symbols less
  Generate symbols during make
  generate_defs: Should run with set -e

BSDmakefile

@@ -5,6 +5,7 @@ MAKE=env -u MAKELEVEL gmake ${MFLAGS}
 TARGETS=	\
 		all \
 		debug \
+		libdefs \
 		static \
 		dynamic \
 		clean \

Makefile.in

@@ -66,7 +66,7 @@ MAKE_UTILS = $(MAKE) 'MAKE=$(MAKE)' 'CXX=$(CXX)' 'CC=$(CC)' 'STRIP=touch' 'INCLU
 'CCDEPMODE=$(CCDEPMODE)' 'DEBLDFLAGS=$(DEBLDFLAGS)' \
 'CXXFLAGS=$(CXXFLAGS) $(DEBCXXFLAGS)' 'CFLGS=$(CFLGS) $(DEBCXXFLAGS)' 'LIBS=$(LIBS)' 'LD=$(LD_DEBUG) -g'
 
-.PHONY: checkclean.static checkclean.dynamic checkclean.debug lib wraith dwraith lib.debug debug test general sorthelp sort indent tar distrib distclean clean cleanutils test all dynamic static tags
+.PHONY: checkclean.static checkclean.dynamic checkclean.debug lib wraith dwraith lib.debug debug test general sorthelp sort indent tar distrib distclean clean cleanutils test all dynamic static tags libdefs
 
 all: dynamic
 
@@ -137,7 +137,38 @@ sort: sorthelp
 sorthelp: src/sorthelp.cc
 	+@cd src && ${MAKE_UTILS} sorthelp
 
-general:
+libdefs: src/.defs/libcrypto_post.h src/.defs/libssl_post.h src/.defs/libtcl_post.h \
+	 src/.defs/libcrypto_pre.h src/.defs/libssl_pre.h src/.defs/libtcl_pre.h \
+	 src/.defs/libcrypto_exports src/.defs/libssl_exports src/.defs/libtcl_exports
+src/.defs/libcrypto_post.h src/.defs/libcrypto_pre.h src/.defs/libcrypto_exports: src/libcrypto.cc \
+	src/generate_defs.sh src/generate_symbol.sh src/dl.h
+	@CXX="$(CXX)" CXXFLAGS="$(CXXFLAGS)" INCLUDES="$(INCLUDES)" \
+           SED="sed" src/generate_defs.sh src/libcrypto.cc || exit 1
+src/.defs/libssl_post.h src/.defs/libssl_pre.h src/.defs/libssl_exports: src/libssl.cc \
+	src/generate_defs.sh src/generate_symbol.sh src/dl.h
+	@CXX="$(CXX)" CXXFLAGS="$(CXXFLAGS)" INCLUDES="$(INCLUDES)" \
+           SED="sed" src/generate_defs.sh src/libssl.cc || exit 1
+src/.defs/libtcl_post.h src/.defs/libtcl_pre.h src/.defs/libtcl_exports: src/libtcl.cc \
+	src/generate_defs.sh src/generate_symbol.sh src/dl.h
+	@CXX="$(CXX)" CXXFLAGS="$(CXXFLAGS)" INCLUDES="$(INCLUDES)" \
+           SED="sed" src/generate_defs.sh src/libtcl.cc || exit 1
+src/.defs/exports: Makefile \
+	src/.defs/libcrypto_exports src/.defs/libssl_exports src/.defs/libtcl_exports
+	@echo "Generating $@"
+	@{ \
+		echo "{"; \
+		echo "bfd_exports_stub;"; \
+		cat src/.defs/*_exports; \
+		echo "};"; \
+	} > $@.tmp
+	@if ! cmp -s $@.tmp $@; then \
+		mv -f $@.tmp $@; \
+	else \
+		rm -f $@.tmp; \
+		touch $@; \
+	fi
+
+general: libdefs src/.defs/exports
 
 .NOTPARALLEL: checkclean.static checkclean.dynamic checkclean.debug
 

build/autotools/configure.ac

@@ -291,6 +291,3 @@ if [[ -n "${LIBELF_BUNDLED}" ]] && [[ -f lib/libelf/configure ]]; then
   AC_CONFIG_SUBDIRS([lib/libelf])
 fi
 AC_OUTPUT
-
-CXX="$CXX" CXXFLAGS="$CXXFLAGS" TCL_INCLUDES="$TCL_INCLUDES" SSL_INCLUDES="$SSL_INCLUDES" \
-  SED="$SED" src/generate_defs.sh || exit 1

configure

@@ -11507,6 +11507,3 @@ printf "%s\n" "$as_me: WARNING: unrecognized options: $ac_unrecognized_opts" >&2
 fi
 
 
-CXX="$CXX" CXXFLAGS="$CXXFLAGS" TCL_INCLUDES="$TCL_INCLUDES" SSL_INCLUDES="$SSL_INCLUDES" \
-  SED="$SED" src/generate_defs.sh || exit 1
-

src/Makefile.in

@@ -113,7 +113,7 @@ sorthelp: sorthelp.cc $(top_builddir)/lib/bdlib/libbdlib.a
 	$(CXX) $(CPPFLAGS) $(CXXFLAGS) $< $(top_builddir)/lib/bdlib/libbdlib.a $(LDFLAGS) -o $@
 	@$(STRIP) $@@EXEEXT@
 
-../$(EGGEXEC): $(OBJS) $(top_builddir)/lib/bdlib/libbdlib.a @LIBELF_BUNDLED@
+../$(EGGEXEC): $(OBJS) $(top_builddir)/lib/bdlib/libbdlib.a @LIBELF_BUNDLED@ $(EXPORTS)
 	@echo -e "[LD ]	\033[1m$@\033[0m"
 	$(LD) $(LDFLAGS) -o ../$(EGGEXEC) $(OBJS) $(top_builddir)/lib/bdlib/libbdlib.a \
 	    $(LIBS) @LIBELF_LIB@ -Wl,--dynamic-list=$(EXPORTS)

src/dl.h

@@ -56,7 +56,7 @@ extern const char *dlsym_error;
 
 extern bd::HashTable<bd::String, FunctionPtr> dl_symbol_table;
 
-#ifdef GENERATE_DEFS
+#ifdef GENERATING_DEFS
 #undef DLSYM_GLOBAL
 #undef DLSYM_GLOBAL_FWDCOMPAT
 #endif

src/generate_defs.sh

@@ -1,4 +1,5 @@
 #! /bin/sh
+set -e
 
 ### Export LC_ALL=C so sort(1) stays consistent
 export LC_ALL=C
@@ -7,31 +8,37 @@ if [ -z "$SED" -o -z "$CXX" ]; then
   echo "This must be ran by configure" >&2
   exit 1
 fi
-echo "==== Generating lib symbols ===="
-INCLUDES="${TCL_INCLUDES} ${SSL_INCLUDES}"
+#echo "==== Generating lib symbols ===="
+: ${INCLUDES:="${TCL_INCLUDES} ${SSL_INCLUDES}"}
 
 mkdir -p src/.defs > /dev/null 2>&1
 TMPFILE=$(mktemp "/tmp/pre.XXXXXX")
-files=$(grep -l DLSYM_GLOBAL src/*.cc|grep -v "src/_")
-exportsFile="src/.defs/exports"
-
-for file in ${files}; do
-  suffix=${file##*.}
-  basename=${file%%.*}
-  basename=${basename##*/}
-
-  defsFile_wrappers="src/.defs/${basename}_defs.${suffix}"
-  defsFile_pre="src/.defs/${basename}_pre.h"
-  defsFile_post="src/.defs/${basename}_post.h"
-
-  rm -f $defsFile_pre $defsFile_post $defsFile_wrappers > /dev/null 2>&1
-  : > $defsFile_pre
-  : > $defsFile_post
-  : > $defsFile_wrappers
-done
+allfiles=$(grep -l DLSYM_GLOBAL src/*.cc|grep -v "src/_")
+
+if [ "$#" -eq 0 ]; then
+	for file in ${allfiles}; do
+	  suffix=${file##*.}
+	  basename=${file%%.*}
+	  basename=${basename##*/}
+
+	  defsFile_wrappers="src/.defs/${basename}_defs.${suffix}"
+	  defsFile_pre="src/.defs/${basename}_pre.h"
+	  defsFile_post="src/.defs/${basename}_post.h"
+
+	  rm -f "${defsFile_pre}" "${defsFile_post}" "${defsFile_wrappers}" > /dev/null 2>&1
+	  if [ ! -f "${defsFile_pre}" ]; then
+		  : > "${defsFile_pre}"
+	  fi
+	  if [ ! -f "${defsFile_post}" ]; then
+		  : > "${defsFile_post}"
+	  fi
+	  if [ ! -f "${defsFile_wrappers}" ]; then
+		  : > "${defsFile_wrappers}"
+	  fi
+	done
+fi
 
-echo "{" > $exportsFile
-echo "bfd_exports_stub;" >> $exportsFile
+: ${files:=${1-$(grep -l DLSYM_GLOBAL src/*.cc|grep -v "src/_")}}
 for file in ${files}; do
   suffix=${file##*.}
   basename=${file%%.*}
@@ -40,24 +47,36 @@ for file in ${files}; do
 
   echo -n "Generating symbols for ${basename}... "
 
+  exportsFile="src/.defs/${basename}_exports"
+  {
+	  #echo "{"
+	  :
+  } > "${exportsFile}.new"
+
   defsFile_wrappers="src/.defs/${basename}_defs.${suffix}"
   defsFile_pre="src/.defs/${basename}_pre.h"
   defsFile_post="src/.defs/${basename}_post.h"
 
-  echo "extern \"C\" {" > $defsFile_wrappers
-  echo "extern \"C\" {" > $defsFile_post
+  {
+	  #echo "#ifndef GENERATING_DEFS"
+	  echo "extern \"C\" {"
+  } > "${defsFile_wrappers}.new"
+  {
+	  #echo "#ifndef GENERATING_DEFS"
+	  echo "extern \"C\" {"
+  } > "${defsFile_post}.new"
 
   cd src
-  $CXX $CXXFLAGS -E -I. -I.. -I../lib ${INCLUDES} -DHAVE_CONFIG_H -DGENERATE_DEFS ../${file} > $TMPFILE
+  $CXX $CXXFLAGS -E -I. -I.. -I../lib ${INCLUDES} -DHAVE_CONFIG_H -DGENERATING_DEFS "../${file}" > "${TMPFILE}"
   # Fix wrapped prototypes
-  $SED -e :a -e N -e '$!ba' -e 's/,\n/,/g' $TMPFILE > $TMPFILE.sed
-  mv $TMPFILE.sed $TMPFILE
+  $SED -e :a -e N -e '$!ba' -e 's/,\n/,/g' "${TMPFILE}" > "${TMPFILE}.sed"
+  mv "${TMPFILE}.sed" "${TMPFILE}"
   cd ..
 
-  $SED -n -e 's/.*\(DLSYM_GLOBAL[^ (]*\)(.*, \([^)]*\).*/\2 \1/p' $TMPFILE | \
+  $SED -n -e 's/.*\(DLSYM_GLOBAL[^ (]*\)(.*, \([^)]*\).*/\2 \1/p' "${TMPFILE}" | \
     sort -u | while read symbol dlsym; do
     # Check if the typedef is already defined ...
-    typedef=$(grep "^typedef .*(\*${symbol}_t)" ${dirname}/${basename}.h)
+    typedef=$(grep "^typedef .*(\*${symbol}_t)" "${dirname}/${basename}.h") || :
     # ... if not, generate it
     if [ -z "$typedef" ]; then
       if ! grep -v "DLSYM" "${TMPFILE}" | grep -qw "${symbol}"; then
@@ -68,7 +87,7 @@ for file in ${files}; do
       fi
 
       # Trim off any extern "C", trim out the variable names, cleanup whitespace issues
-      typedef=$(grep -w "${symbol}" $TMPFILE |
+      typedef=$(grep -w "${symbol}" "${TMPFILE}" |
 	      head -n 1 |
 	      $SED \
 	         -e 's/extern "C" *//' \
@@ -78,7 +97,7 @@ for file in ${files}; do
 		 -e 's/  */ /g' \
 		 -e 's/ \([,)]\)/\1/g' \
 		 -e 's/ *()/(void)/g' \
-	 )
+	 ) || :
       existing_typedef=0
     else
       existing_typedef=1
@@ -86,22 +105,52 @@ for file in ${files}; do
 
     if [ "${typedef%;}" = "${typedef}" ]; then
       echo "Error: Unable to generate typedef for: ${symbol}" >&2
-      test -n "$typedef" && echo "$typedef" >&2
+      if [ -n "${typedef}" ]; then
+	      echo "$typedef" >&2
+      fi
       continue
     fi
 
     #pipe typedef into generate_symbol.sh
-    [ -z "$typedef" ] && continue
+    if [ -z "${typedef}" ]; then
+	    continue
+    fi
     if [ "${dlsym}" = "DLSYM_GLOBAL_FWDCOMPAT" ]; then
-      echo "_${symbol};" >> $exportsFile
+      echo "_${symbol};" >> "${exportsFile}.new"
     fi
     echo "${symbol} ${existing_typedef} ${typedef}"
-  done | src/generate_symbol.sh $defsFile_wrappers $defsFile_pre $defsFile_post
-
-  echo "}" >> $defsFile_wrappers
-  echo "}" >> $defsFile_post
-
-  echo "done"
+  done | src/generate_symbol.sh "${defsFile_wrappers}" "${defsFile_pre}" "${defsFile_post}"
+
+  {
+	  echo "}"
+	  #echo "#endif"
+  } >> "${defsFile_wrappers}.new"
+  {
+	  echo "}"
+	  #echo "#endif"
+  } >> "${defsFile_post}.new"
+
+  {
+	  :
+	  #echo "};"
+  } >> "${exportsFile}.new"
+
+  changed=0
+  for file in ${defsFile_wrappers} ${defsFile_pre} ${defsFile_post} ${exportsFile}; do
+	  if ! cmp -s "${file}.new" "${file}"; then
+		  changed=1
+		  mv -f "${file}.new" "${file}"
+	  else
+		  rm -f "${file}.new"
+		  # Needed because generate_defs.sh may have triggered this
+		  # (or other deps like dl.h)
+		  touch "${file}"
+	  fi
+  done
+  if [ "${changed}" -eq 1 ]; then
+	  echo "done"
+  else
+	  echo "unchanged"
+  fi
 done
-echo "};">> $exportsFile
-rm -f $TMPFILE
+rm -f "${TMPFILE}"

src/generate_symbol.sh

@@ -1,4 +1,5 @@
 #! /bin/sh
+set -e
 
 defsFile_wrappers="$1"
 defsFile_pre="$2"
@@ -6,14 +7,17 @@ defsFile_post="$3"
 
 # X="typedef int (*Tcl_Eval_t)(Tcl_Interp*, const char*);"
 
+echo "#ifndef GENERATING_DEFS" > "${defsFile_pre}.new"
 while read symbol existing_typedef typedef; do
-  echo "#define ${symbol} ORIGINAL_SYMBOL_${symbol}" >> $defsFile_pre
-  echo "#undef ${symbol}" >> $defsFile_post
-  test $existing_typedef -eq 0 && echo "$typedef" >> $defsFile_post
+  echo "#define ${symbol} ORIGINAL_SYMBOL_${symbol}" >> "${defsFile_pre}.new"
+  echo "#undef ${symbol}" >> "${defsFile_post}.new"
+  if [ "${existing_typedef}" -eq 0 ]; then
+	  echo "$typedef" >> "${defsFile_post}.new"
+  fi
 
-  returntype=$(echo "$typedef" | sed -e 's/typedef \([^(]*\) (\*\([^ ]*\)_t)(\(.*\));/\1/')
-  name=$(echo "$typedef" | sed -e 's/typedef \([^(]*\) (\*\([^ ]*\)_t)(\(.*\));/\2/')
-  params=$(echo "$typedef" | sed -e 's/typedef \([^(]*\) (\*\([^ ]*\)_t)(\(.*\));/\3/')
+  returntype=$(echo "$typedef" | sed -e 's/typedef \([^(]*\) (\*\([^ ]*\)_t)(\(.*\));/\1/') || :
+  name=$(echo "$typedef" | sed -e 's/typedef \([^(]*\) (\*\([^ ]*\)_t)(\(.*\));/\2/') || :
+  params=$(echo "$typedef" | sed -e 's/typedef \([^(]*\) (\*\([^ ]*\)_t)(\(.*\));/\3/') ||:
 
   SAVE_IFS=$IFS
   IFS=,
@@ -24,7 +28,7 @@ while read symbol existing_typedef typedef; do
   # Set params to $1,$2, etc
   set $params
   paramCount=$#
-  lastParam=$(expr $paramCount - 1)
+  lastParam=$((paramCount - 1))
   i=0
 
   while [ $i -lt $paramCount ]; do
@@ -44,13 +48,14 @@ while read symbol existing_typedef typedef; do
     i=$((i + 1))
   done
 
-  cat >> $defsFile_wrappers << EOF
+  cat >> "${defsFile_wrappers}.new" << EOF
 $returntype $name ($params_full) {
   return DLSYM_VAR($name)($param_names);
 }
 EOF
 
-  echo "$returntype $name ($params_full);" >> $defsFile_post
+  echo "$returntype $name ($params_full);" >> "${defsFile_post}.new"
 
   IFS=$SAVE_IFS
 done
+echo "#endif" >> "${defsFile_pre}.new"

src/libcrypto.cc

@@ -33,7 +33,9 @@
 #include <bdlib/src/Array.h>
 
 #include "libcrypto.h"
+#ifndef GENERATING_DEFS
 #include ".defs/libcrypto_defs.cc"
+#endif
 
 #ifndef OPENSSL_SHLIB_VERSION
 #define OPENSSL_SHLIB_VERSION_STR SHLIB_VERSION_NUMBER

src/libcrypto.h

@@ -5,7 +5,9 @@
 #  include "config.h"
 #endif
 
+#ifndef GENERATING_DEFS
 #include ".defs/libcrypto_pre.h"
+#endif
 
 #include <openssl/crypto.h>
 #include <openssl/aes.h>
@@ -16,7 +18,9 @@
 #include <openssl/err.h>
 #include <openssl/rand.h>
 
+#ifndef GENERATING_DEFS
 #include ".defs/libcrypto_post.h"
+#endif
 
 #include "src/crypto/aes_util.h"
 #include "src/crypto/bf_util.h"

src/libssl.cc

@@ -33,7 +33,9 @@
 #include <bdlib/src/Array.h>
 
 #include "libssl.h"
+#ifndef GENERATING_DEFS
 #include ".defs/libssl_defs.cc"
+#endif
 
 #ifndef OPENSSL_SHLIB_VERSION
 #define OPENSSL_SHLIB_VERSION_STR SHLIB_VERSION_NUMBER

src/libssl.h

@@ -9,7 +9,9 @@
 #include "dl.h"
 #include <bdlib/src/String.h>
 
+#ifndef GENERATING_DEFS
 #include ".defs/libssl_pre.h"
+#endif
 
 #ifdef EGG_SSL_EXT
 # ifndef EGG_SSL_INCS
@@ -20,7 +22,9 @@
 
 typedef DH* (*dh_callback_t)(SSL*, int, int);
 
+#ifndef GENERATING_DEFS
 #include ".defs/libssl_post.h"
+#endif
 
 typedef void (*SSL_CTX_set_tmp_dh_callback_t)(SSL_CTX*, dh_callback_t);
 

src/libtcl.cc

@@ -32,7 +32,9 @@
 #include <bdlib/src/Array.h>
 
 #include "libtcl.h"
+#ifndef GENERATING_DEFS
 #include ".defs/libtcl_defs.cc"
+#endif
 
 #ifdef USE_SCRIPT_TCL
 Tcl_Interp *global_interp = NULL;

src/libtcl.h

@@ -6,11 +6,15 @@
 #include <bdlib/src/String.h>
 #ifdef USE_SCRIPT_TCL
 
+#ifndef GENERATING_DEFS
 #include ".defs/libtcl_pre.h"
+#endif
 
 #include <tcl.h>
 
+#ifndef GENERATING_DEFS
 #include ".defs/libtcl_post.h"
+#endif
 
 
 #define STDVAR (ClientData cd, Tcl_Interp *interp, int argc, const char *argv[])