* Use bd::base64(Encode|Decode) with helper functions to match FiSH base64 encoding

src/crypto/dh_util.c

@@ -9,80 +9,6 @@
 #include <bdlib/src/base64.h>
 #include "dh_util.h"
 
-/*
-   int b64toh(lpBase64String, lpDestinationBuffer);
-   Converts base64 string b to hexnumber d.
-   Returns size of hexnumber in bytes.
-   */
-int b64toh(char *b, char *d){
-  int i,k,l;
-
-  l=strlen(b);
-  if (l<2) return 0;
-  for (i=l-1;i>-1;i--){
-    if (bd::b64_indexes[(unsigned char)(b[i])]==0) l--;
-    else break;
-  }
-
-  if (l<2) return 0;
-  i=0, k=0;
-  while (1) {
-    i++;
-    if (k+1<l) d[i-1]=((bd::b64_indexes[(unsigned char)(b[k])])<<2);
-    else break;
-    k++;
-    if (k<l) d[i-1]|=((bd::b64_indexes[(unsigned char)(b[k])])>>4);
-    else break;
-    i++;
-    if (k+1<l) d[i-1]=((bd::b64_indexes[(unsigned char)(b[k])])<<4);
-    else break;
-    k++;
-    if (k<l) d[i-1]|=((bd::b64_indexes[(unsigned char)(b[k])])>>2);
-    else break;
-    i++;
-    if (k+1<l) d[i-1]=((bd::b64_indexes[(unsigned char)(b[k])])<<6);
-    else break;
-    k++;
-    if (k<l) d[i-1]|=(bd::b64_indexes[(unsigned char)(b[k])]);
-    else break;
-    k++;
-  }
-  return i-1;
-}
-
-/*
-   int htob64(lpHexNumber, lpDestinationBuffer);
-   Converts hexnumber h (with length l bytes) to base64 string d.
-   Returns length of base64 string.
-   */
-int htob64(char *h, char *d, unsigned int l){
-  unsigned int i,j,k;
-  unsigned char m,t;
-
-  if (!l) return 0;
-  l<<=3;                              // no. bits
-  m=0x80;
-  for (i=0,j=0,k=0,t=0; i<l; i++){
-    if (h[(i>>3)]&m) t|=1;
-    j++;
-    if (!(m>>=1)) m=0x80;
-    if (!(j%6)) {
-      d[k]=bd::b64_charset[t];
-      t&=0;
-      k++;
-    }
-    t<<=1;
-  }
-  m=5-(j%6);
-  t<<=m;
-  if (m) {
-    d[k]=bd::b64_charset[t];
-    k++;
-  }
-  d[k]&=0;
-  return strlen(d);
-}
-
 static BIGNUM* b_prime = NULL;
 static BIGNUM* b_generator = NULL;
 
@@ -104,6 +30,50 @@ void DH1080_init() {
   }
 }
 
+/**
+ * @brief Encode a string using FiSH's base64 algorithm (from FiSH/mIRC)
+ * @note Any = padding is removed, and an 'A' is added if no padding was needed
+ * @param bd::String str The string to encode
+ * @returns Encoded string
+ * @note Adapated from FiSH code
+ */
+bd::String fishBase64Encode(const bd::String& str) {
+  bd::String result(bd::base64Encode(str));
+
+  // No padding, add an A on the end (base64-encoded NULL-terminator)
+  if (result.rfind('=') == result.npos) {
+    result += 'A';
+  } else {
+    // Remove padding
+    while (result.rfind('=') != result.npos) {
+      --result;
+    }
+  }
+  return result;
+}
+
+/**
+ * @brief Decode a string using FiSH's base64 algorithm (from FiSH/mIRC)
+ * @param bd::String str The string to decode
+ * @returns Decoded data
+ * @note Adapated from FiSH code
+ */
+bd::String fishBase64Decode(const bd::String& str) {
+  bd::String temp(str);
+
+  // Remove the 'A' NULL-terminator if present
+  if (temp.length() % 4 == 1 && temp(-1, 1) == 'A') {
+    --temp;
+  }
+
+  while (temp.length() % 4) {
+    temp += '=';
+  }
+
+  return bd::base64Decode(temp);
+}
+
+
 void DH1080_gen(bd::String& privateKey, bd::String& publicKeyB64) {
   DH *dh = NULL;
 
@@ -127,14 +97,12 @@ void DH1080_gen(bd::String& privateKey, bd::String& publicKeyB64) {
   BN_bn2bin(dh->pub_key, reinterpret_cast<unsigned char*>(publicKey.mdata()));;
 
   // base64 encode
-  publicKeyB64 = bd::base64Encode(publicKey);
+  publicKeyB64 = fishBase64Encode(publicKey);
 
   DH_free(dh);
 }
 
 bool DH1080_comp(const bd::String privateKey, const bd::String theirPublicKeyB64, bd::String& sharedKey) {
-  size_t len = 0;
-  unsigned char raw_buf[200] = "";
   BIGNUM *b_myPrivkey = NULL, *b_HisPubkey = NULL;
   DH *dh = NULL;
 
@@ -148,13 +116,12 @@ bool DH1080_comp(const bd::String privateKey, const bd::String theirPublicKeyB64
   dh->priv_key = b_myPrivkey;
 
   // Prep their public key
-  len = theirPublicKeyB64.length();
-  bd::b64dec_buf(reinterpret_cast<const unsigned char*>(theirPublicKeyB64.data()), &len, reinterpret_cast<char*>(raw_buf));
-  b_HisPubkey = BN_bin2bn(reinterpret_cast<const unsigned char*>(raw_buf), len, NULL);
+  bd::String theirPublicKey(fishBase64Decode(theirPublicKeyB64));
+  b_HisPubkey = BN_bin2bn(reinterpret_cast<const unsigned char*>(theirPublicKey.data()), theirPublicKey.length(), NULL);
 
   // Compute the Shared key
   char *key = (char *)my_calloc(1, DH_size(dh));
-  len = DH_compute_key((unsigned char *)key, b_HisPubkey, dh);
+  size_t len = DH_compute_key((unsigned char *)key, b_HisPubkey, dh);
   DH_free(dh);
   BN_clear_free(b_HisPubkey);
   if (len == static_cast<size_t>(-1)) {
@@ -168,14 +135,13 @@ bool DH1080_comp(const bd::String privateKey, const bd::String theirPublicKeyB64
   }
 
   SHA256_CTX c;
-  unsigned char SHA256digest[SHA256_DIGEST_LENGTH] = "";
+  bd::String SHA256Digest(static_cast<size_t>(SHA256_DIGEST_LENGTH));
+  SHA256Digest.resize(SHA256_DIGEST_LENGTH);
 
   SHA256_Init(&c);
   SHA256_Update(&c, key, len);
-  SHA256_Final(SHA256digest, &c);
-  memset(raw_buf, 0, sizeof(raw_buf));
-  len = htob64((char *)SHA256digest, (char *)raw_buf, sizeof(SHA256digest));
-  sharedKey = bd::String(reinterpret_cast<char *>(raw_buf), len);
+  SHA256_Final(reinterpret_cast<unsigned char*>(SHA256Digest.mdata()), &c);
+  sharedKey = fishBase64Encode(SHA256Digest);
 
   free(key);
 

src/crypto/dh_util.h

@@ -13,6 +13,8 @@ namespace bd {
 }
 
 // Adapated from znc-fish
+bd::String fishBase64Encode(const bd::String& str);
+bd::String fishBase64Decode(const bd::String& str);
 void DH1080_gen(bd::String& privateKey, bd::String& publicKeyB64);
 bool DH1080_comp(const bd::String privateKey, const bd::String theirPublicKeyB64, bd::String& sharedKey);
 void DH1080_init();

src/mod/server.mod/servmsg.c

@@ -890,17 +890,9 @@ static int gotnotice(char *from, char *msg)
 
         if (!strncmp(msg, "DH1080_INIT ", 12)) {
           bd::String theirPublicKeyB64(msg + 12);
-          // Some FiSH implementations improperly encode their NULL terminator (A) on the end, just trim it off.
-          if (theirPublicKeyB64(-1, 1) == 'A' && theirPublicKeyB64.length() == 181) {
-            theirPublicKeyB64.resize(180, 0);
-          }
           handle_DH1080_init(nick, uhost, from, u, theirPublicKeyB64);
         } else if (!strncmp(msg, "DH1080_FINISH ", 14)) {
           bd::String theirPublicKeyB64(msg + 14);
-          // Some FiSH implementations improperly encode their NULL terminator (A) on the end, just trim it off.
-          if (theirPublicKeyB64(-1, 1) == 'A' && theirPublicKeyB64.length() == 181) {
-            theirPublicKeyB64.resize(180, 0);
-          }
           handle_DH1080_finish(nick, uhost, from, u, theirPublicKeyB64);
         } else {
           putlog(LOG_MSGS, "*", "-%s (%s)- %s", nick, uhost, msg);