diff -u -r -N squid-4.8/acinclude/compiler-flags.m4 squid-4.9/acinclude/compiler-flags.m4 --- squid-4.8/acinclude/compiler-flags.m4 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/acinclude/compiler-flags.m4 2019-11-06 08:14:40.000000000 +1300 @@ -41,7 +41,7 @@ SAVED_CFLAGS="$CFLAGS" SAVED_CXXFLAGS="$CXXFLAGS" AC_COMPILE_IFELSE([AC_LANG_PROGRAM($3,$4)],[$1=no],[],[$1=no]) - if test "x$1" != "xno" ; then + if test "x$$1" != "xno" ; then CFLAGS="$CFLAGS $2" CXXFLAGS="$CXXFLAGS $2" AC_COMPILE_IFELSE([AC_LANG_PROGRAM($3,$4)],[$1=yes],[$1=no],[$1=no]) diff -u -r -N squid-4.8/acinclude/os-deps.m4 squid-4.9/acinclude/os-deps.m4 --- squid-4.8/acinclude/os-deps.m4 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/acinclude/os-deps.m4 2019-11-06 08:14:40.000000000 +1300 @@ -231,7 +231,7 @@ fprintf (fp, "%d\n", i & ~0x3F); return 0; } - ]])],[squid_filedescriptors_limit=`cat conftestval`],[],[]) + ]])],[squid_filedescriptors_limit=`cat conftestval`],[],[:]) dnl Microsoft MSVCRT.DLL supports 2048 maximum FDs AS_CASE(["$host_os"],[mingw|mingw32],[squid_filedescriptors_limit="2048"]) AC_MSG_RESULT($squid_filedescriptors_limit) diff -u -r -N squid-4.8/acinclude/pam.m4 squid-4.9/acinclude/pam.m4 --- squid-4.8/acinclude/pam.m4 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/acinclude/pam.m4 2019-11-06 08:14:40.000000000 +1300 @@ -21,7 +21,7 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ #include static int -password_conversation(int num_msg, const struct pam_message **msg, struct pam_response **resp, void *appdata_ptr) {} +password_conversation(int num_msg, const struct pam_message **msg, struct pam_response **resp, void *appdata_ptr) { return 0; } static struct pam_conv conv = { &password_conversation, 0 }; ]])], [ squid_cv_pam_conv_signature=linux @@ -29,7 +29,7 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ #include static int -password_conversation(int num_msg, struct pam_message **msg, struct pam_response **resp, void *appdata_ptr) {} +password_conversation(int num_msg, struct pam_message **msg, struct pam_response **resp, void *appdata_ptr) { return 0; } static struct pam_conv conv = { &password_conversation, 0 }; ]])], [ squid_cv_pam_conv_signature=solaris diff -u -r -N squid-4.8/cfgaux/ltmain.sh squid-4.9/cfgaux/ltmain.sh --- squid-4.8/cfgaux/ltmain.sh 2019-07-10 07:16:44.000000000 +1200 +++ squid-4.9/cfgaux/ltmain.sh 2019-11-06 08:18:59.000000000 +1300 @@ -31,7 +31,7 @@ PROGRAM=libtool PACKAGE=libtool -VERSION="2.4.6 Debian-2.4.6-10" +VERSION="2.4.6 Debian-2.4.6-11" package_revision=2.4.6 @@ -2141,7 +2141,7 @@ compiler: $LTCC compiler flags: $LTCFLAGS linker: $LD (gnu? $with_gnu_ld) - version: $progname $scriptversion Debian-2.4.6-10 + version: $progname $scriptversion Debian-2.4.6-11 automake: `($AUTOMAKE --version) 2>/dev/null |$SED 1q` autoconf: `($AUTOCONF --version) 2>/dev/null |$SED 1q` diff -u -r -N squid-4.8/ChangeLog squid-4.9/ChangeLog --- squid-4.8/ChangeLog 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/ChangeLog 2019-11-06 08:14:40.000000000 +1300 @@ -1,3 +1,27 @@ +Changes to squid-4.9 (05 Nov 2019): + + - Bug 4978: eCAP crash after using MyHost().newRequest() + - Bug 4970: excessive gnutls_certificate_credentials debug msgs + - Bug 4969: GCC-9 build failure: stringop-truncation + - Bug 4966: Lower cache_peer hostname + - Bug 4918: Crashes when using OpenSSL prior to v1.0.2 + - TLS: Fix parsing of certificate validator responses + - TLS: Fix parsing of TLS messages that span multiple records + - TLS: Fix on_unsupported_protocol tunnel action + - TLS: Fix expiration of self-signed generated certs to be 3 years + - HTTP: Ignore malformed Host header in intercept and reverse proxy mode + - HTTP: RFC 7230: server MUST reject messages with BWS after field-name + - HTTP: Fix URN response handling + - HTTP: Hash Digest noncedata + - Update URI parser to use SBuf parsing APIs + - Prevent truncation for large origin-relative domains + - Fix several rock cache_dir corruption issues + - Debug detail validation errors for loaded-from-file certificate chains + - smblib: Improve SMB server name maintenance + - cachemgr.cgi: Add validation for hostname parameter + - ... and several compile issues + - ... and some documentation updates + Changes to squid-4.8 (09 Jul 2019): - Bug 4957: Multiple XSS issues in cachemgr.cgi diff -u -r -N squid-4.8/compat/openssl.h squid-4.9/compat/openssl.h --- squid-4.8/compat/openssl.h 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/compat/openssl.h 2019-11-06 08:14:40.000000000 +1300 @@ -177,9 +177,9 @@ X509_get0_signature(ASN1_BIT_STRING **psig, X509_ALGOR **palg, const X509 *x) { if (psig) - *psig = (ASN1_BIT_STRING *)&x->signature; + *psig = x->signature; if (palg) - *palg = (X509_ALGOR *)&x->sig_alg; + *palg = x->sig_alg; } #endif diff -u -r -N squid-4.8/configure squid-4.9/configure --- squid-4.8/configure 2019-07-10 07:16:52.000000000 +1200 +++ squid-4.9/configure 2019-11-06 08:19:06.000000000 +1300 @@ -1,7 +1,7 @@ #! /bin/sh # From configure.ac Revision. # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for Squid Web Proxy 4.8. +# Generated by GNU Autoconf 2.69 for Squid Web Proxy 4.9. # # Report bugs to . # @@ -595,8 +595,8 @@ # Identity of this package. PACKAGE_NAME='Squid Web Proxy' PACKAGE_TARNAME='squid' -PACKAGE_VERSION='4.8' -PACKAGE_STRING='Squid Web Proxy 4.8' +PACKAGE_VERSION='4.9' +PACKAGE_STRING='Squid Web Proxy 4.9' PACKAGE_BUGREPORT='http://bugs.squid-cache.org/' PACKAGE_URL='' @@ -1651,7 +1651,7 @@ # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures Squid Web Proxy 4.8 to adapt to many kinds of systems. +\`configure' configures Squid Web Proxy 4.9 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1722,7 +1722,7 @@ if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of Squid Web Proxy 4.8:";; + short | recursive ) echo "Configuration of Squid Web Proxy 4.9:";; esac cat <<\_ACEOF @@ -2155,7 +2155,7 @@ test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -Squid Web Proxy configure 4.8 +Squid Web Proxy configure 4.9 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -3259,7 +3259,7 @@ This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by Squid Web Proxy $as_me 4.8, which was +It was created by Squid Web Proxy $as_me 4.9, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -4126,7 +4126,7 @@ # Define the identity of the package. PACKAGE='squid' - VERSION='4.8' + VERSION='4.9' cat >>confdefs.h <<_ACEOF @@ -20832,7 +20832,7 @@ ac_cv_require_wno_deprecated_register=no fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext - if test "xac_cv_require_wno_deprecated_register" != "xno" ; then + if test "x$ac_cv_require_wno_deprecated_register" != "xno" ; then CFLAGS="$CFLAGS -Werror -Wno-deprecated-register" CXXFLAGS="$CXXFLAGS -Werror -Wno-deprecated-register" cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -23992,9 +23992,9 @@ fi - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for SSL_library_init in -lssl" >&5 -$as_echo_n "checking for SSL_library_init in -lssl... " >&6; } -if ${ac_cv_lib_ssl_SSL_library_init+:} false; then : + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for SSL_CTX_new in -lssl" >&5 +$as_echo_n "checking for SSL_CTX_new in -lssl... " >&6; } +if ${ac_cv_lib_ssl_SSL_CTX_new+:} false; then : $as_echo_n "(cached) " >&6 else ac_check_lib_save_LIBS=$LIBS @@ -24008,27 +24008,27 @@ #ifdef __cplusplus extern "C" #endif -char SSL_library_init (); +char SSL_CTX_new (); int main () { -return SSL_library_init (); +return SSL_CTX_new (); ; return 0; } _ACEOF if ac_fn_cxx_try_link "$LINENO"; then : - ac_cv_lib_ssl_SSL_library_init=yes + ac_cv_lib_ssl_SSL_CTX_new=yes else - ac_cv_lib_ssl_SSL_library_init=no + ac_cv_lib_ssl_SSL_CTX_new=no fi rm -f core conftest.err conftest.$ac_objext \ conftest$ac_exeext conftest.$ac_ext LIBS=$ac_check_lib_save_LIBS fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_ssl_SSL_library_init" >&5 -$as_echo "$ac_cv_lib_ssl_SSL_library_init" >&6; } -if test "x$ac_cv_lib_ssl_SSL_library_init" = xyes; then : +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_ssl_SSL_CTX_new" >&5 +$as_echo "$ac_cv_lib_ssl_SSL_CTX_new" >&6; } +if test "x$ac_cv_lib_ssl_SSL_CTX_new" = xyes; then : LIBOPENSSL_LIBS="-lssl $LIBOPENSSL_LIBS" else @@ -24095,9 +24095,9 @@ fi - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for SSL_library_init in -lssl" >&5 -$as_echo_n "checking for SSL_library_init in -lssl... " >&6; } -if ${ac_cv_lib_ssl_SSL_library_init+:} false; then : + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for SSL_CTX_new in -lssl" >&5 +$as_echo_n "checking for SSL_CTX_new in -lssl... " >&6; } +if ${ac_cv_lib_ssl_SSL_CTX_new+:} false; then : $as_echo_n "(cached) " >&6 else ac_check_lib_save_LIBS=$LIBS @@ -24111,27 +24111,27 @@ #ifdef __cplusplus extern "C" #endif -char SSL_library_init (); +char SSL_CTX_new (); int main () { -return SSL_library_init (); +return SSL_CTX_new (); ; return 0; } _ACEOF if ac_fn_cxx_try_link "$LINENO"; then : - ac_cv_lib_ssl_SSL_library_init=yes + ac_cv_lib_ssl_SSL_CTX_new=yes else - ac_cv_lib_ssl_SSL_library_init=no + ac_cv_lib_ssl_SSL_CTX_new=no fi rm -f core conftest.err conftest.$ac_objext \ conftest$ac_exeext conftest.$ac_ext LIBS=$ac_check_lib_save_LIBS fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_ssl_SSL_library_init" >&5 -$as_echo "$ac_cv_lib_ssl_SSL_library_init" >&6; } -if test "x$ac_cv_lib_ssl_SSL_library_init" = xyes; then : +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_ssl_SSL_CTX_new" >&5 +$as_echo "$ac_cv_lib_ssl_SSL_CTX_new" >&6; } +if test "x$ac_cv_lib_ssl_SSL_CTX_new" = xyes; then : LIBOPENSSL_LIBS="-lssl $LIBOPENSSL_LIBS" else @@ -34760,7 +34760,7 @@ #include static int -password_conversation(int num_msg, const struct pam_message **msg, struct pam_response **resp, void *appdata_ptr) {} +password_conversation(int num_msg, const struct pam_message **msg, struct pam_response **resp, void *appdata_ptr) { return 0; } static struct pam_conv conv = { &password_conversation, 0 }; int @@ -34782,7 +34782,7 @@ #include static int -password_conversation(int num_msg, struct pam_message **msg, struct pam_response **resp, void *appdata_ptr) {} +password_conversation(int num_msg, struct pam_message **msg, struct pam_response **resp, void *appdata_ptr) { return 0; } static struct pam_conv conv = { &password_conversation, 0 }; int @@ -40473,10 +40473,7 @@ ;; esac if test "$cross_compiling" = yes; then : - { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error $? "cannot run test program while cross compiling -See \`config.log' for more details" "$LINENO" 5; } + : else cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -40976,7 +40973,7 @@ ac_cv_require_rtti=no fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext - if test "xac_cv_require_rtti" != "xno" ; then + if test "x$ac_cv_require_rtti" != "xno" ; then CFLAGS="$CFLAGS -rtti" CXXFLAGS="$CXXFLAGS -rtti" cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -41055,7 +41052,7 @@ ac_cv_require_qcpluscmt=no fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext - if test "xac_cv_require_qcpluscmt" != "xno" ; then + if test "x$ac_cv_require_qcpluscmt" != "xno" ; then CFLAGS="$CFLAGS -qcpluscmt" CXXFLAGS="$CXXFLAGS -qcpluscmt" cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -44215,7 +44212,7 @@ # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by Squid Web Proxy $as_me 4.8, which was +This file was extended by Squid Web Proxy $as_me 4.9, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -44281,7 +44278,7 @@ cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -Squid Web Proxy config.status 4.8 +Squid Web Proxy config.status 4.9 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff -u -r -N squid-4.8/configure.ac squid-4.9/configure.ac --- squid-4.8/configure.ac 2019-07-10 07:16:52.000000000 +1200 +++ squid-4.9/configure.ac 2019-11-06 08:19:06.000000000 +1300 @@ -5,7 +5,7 @@ ## Please see the COPYING and CONTRIBUTORS files for details. ## -AC_INIT([Squid Web Proxy],[4.8],[http://bugs.squid-cache.org/],[squid]) +AC_INIT([Squid Web Proxy],[4.9],[http://bugs.squid-cache.org/],[squid]) AC_PREREQ(2.61) AC_CONFIG_HEADERS([include/autoconf.h]) AC_CONFIG_AUX_DIR(cfgaux) @@ -1357,7 +1357,7 @@ AC_CHECK_LIB(crypto,[CRYPTO_new_ex_data],[LIBOPENSSL_LIBS="-lcrypto $LIBOPENSSL_LIBS"],[ AC_MSG_ERROR([library 'crypto' is required for OpenSSL]) ],$LIBOPENSSL_LIBS) - AC_CHECK_LIB(ssl,[SSL_library_init],[LIBOPENSSL_LIBS="-lssl $LIBOPENSSL_LIBS"],[ + AC_CHECK_LIB(ssl,[SSL_CTX_new],[LIBOPENSSL_LIBS="-lssl $LIBOPENSSL_LIBS"],[ AC_MSG_ERROR([library 'ssl' is required for OpenSSL]) ],$LIBOPENSSL_LIBS) ]) diff -u -r -N squid-4.8/doc/release-notes/release-4.html squid-4.9/doc/release-notes/release-4.html --- squid-4.8/doc/release-notes/release-4.html 2019-07-10 07:25:09.000000000 +1200 +++ squid-4.9/doc/release-notes/release-4.html 2019-11-06 08:27:21.000000000 +1300 @@ -2,10 +2,10 @@ - Squid 4.8 release notes + Squid 4.9 release notes -

Squid 4.8 release notes

+

Squid 4.9 release notes

Squid Developers

@@ -63,7 +63,7 @@

1. Notice

-

The Squid Team are pleased to announce the release of Squid-4.8 for testing.

+

The Squid Team are pleased to announce the release of Squid-4.9 for testing.

This new release is available for download from http://www.squid-cache.org/Versions/v4/ or the mirrors.

diff -u -r -N squid-4.8/include/version.h squid-4.9/include/version.h --- squid-4.8/include/version.h 2019-07-10 07:16:52.000000000 +1200 +++ squid-4.9/include/version.h 2019-11-06 08:19:06.000000000 +1300 @@ -7,7 +7,7 @@ */ #ifndef SQUID_RELEASE_TIME -#define SQUID_RELEASE_TIME 1562699800 +#define SQUID_RELEASE_TIME 1572981533 #endif /* diff -u -r -N squid-4.8/lib/smblib/smblib.c squid-4.9/lib/smblib/smblib.c --- squid-4.8/lib/smblib/smblib.c 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/lib/smblib/smblib.c 2019-11-06 08:14:40.000000000 +1300 @@ -149,10 +149,10 @@ /* Now connect to the remote end, but first upper case the name of the service we are going to call, sine some servers want it in uppercase */ - for (i=0; i < strlen(server); i++) - called[i] = xtoupper(server[i]); + for (i=0; i < strlen(con -> desthost); i++) + called[i] = xtoupper(con -> desthost[i]); - called[strlen(server)] = 0; /* Make it a string */ + called[strlen(con -> desthost)] = 0; /* Make it a string */ for (i=0; i < strlen(con -> myname); i++) calling[i] = xtoupper(con -> myname[i]); @@ -265,10 +265,10 @@ /* Now connect to the remote end, but first upper case the name of the service we are going to call, sine some servers want it in uppercase */ - for (i=0; i < strlen(host); i++) - called[i] = xtoupper(host[i]); + for (i=0; i < strlen(con -> desthost); i++) + called[i] = xtoupper(con -> desthost[i]); - called[strlen(host)] = 0; /* Make it a string */ + called[strlen(con -> desthost)] = 0; /* Make it a string */ for (i=0; i < strlen(con -> myname); i++) calling[i] = xtoupper(con -> myname[i]); diff -u -r -N squid-4.8/lib/smblib/smblib-util.c squid-4.9/lib/smblib/smblib-util.c --- squid-4.8/lib/smblib/smblib-util.c 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/lib/smblib/smblib-util.c 2019-11-06 08:14:40.000000000 +1300 @@ -401,7 +401,7 @@ p = (SMB_Hdr(pkt) + SMB_negrLM_buf_offset + Con_Handle -> Encrypt_Key_Len); - strncpy(p, Con_Handle -> Svr_PDom, sizeof(Con_Handle -> Svr_PDom) - 1); + xstrncpy(p, Con_Handle -> Svr_PDom, sizeof(Con_Handle -> Svr_PDom)); break; @@ -424,7 +424,7 @@ p = (SMB_Hdr(pkt) + SMB_negrLM_buf_offset + Con_Handle -> Encrypt_Key_Len); - strncpy(p, Con_Handle -> Svr_PDom, sizeof(Con_Handle -> Svr_PDom) - 1); + xstrncpy(p, Con_Handle -> Svr_PDom, sizeof(Con_Handle -> Svr_PDom)); break; @@ -538,8 +538,8 @@ tree -> next = tree -> prev = NULL; tree -> con = Con_Handle; - strncpy(tree -> path, path, sizeof(tree -> path)); - strncpy(tree -> device_type, device, sizeof(tree -> device_type)); + xstrncpy(tree -> path, path, sizeof(tree -> path)); + xstrncpy(tree -> device_type, device, sizeof(tree -> device_type)); /* Now plug in the values ... */ diff -u -r -N squid-4.8/RELEASENOTES.html squid-4.9/RELEASENOTES.html --- squid-4.8/RELEASENOTES.html 2019-07-10 07:25:09.000000000 +1200 +++ squid-4.9/RELEASENOTES.html 2019-11-06 08:27:21.000000000 +1300 @@ -2,10 +2,10 @@ - Squid 4.8 release notes + Squid 4.9 release notes -

Squid 4.8 release notes

+

Squid 4.9 release notes

Squid Developers

@@ -63,7 +63,7 @@

1. Notice

-

The Squid Team are pleased to announce the release of Squid-4.8 for testing.

+

The Squid Team are pleased to announce the release of Squid-4.9 for testing.

This new release is available for download from http://www.squid-cache.org/Versions/v4/ or the mirrors.

diff -u -r -N squid-4.8/src/acl/Asn.cc squid-4.9/src/acl/Asn.cc --- squid-4.8/src/acl/Asn.cc 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/src/acl/Asn.cc 2019-11-06 08:14:40.000000000 +1300 @@ -243,7 +243,7 @@ snprintf(asres, 4096, "whois://%s/!gAS%d", Config.as_whois_server, as); asState->as_number = as; const MasterXaction::Pointer mx = new MasterXaction(XactionInitiator::initAsn); - asState->request = HttpRequest::FromUrl(asres, mx); + asState->request = HttpRequest::FromUrlXXX(asres, mx); assert(asState->request != NULL); if ((e = storeGetPublic(asres, Http::METHOD_GET)) == NULL) { diff -u -r -N squid-4.8/src/acl/external/delayer/ext_delayer_acl.8 squid-4.9/src/acl/external/delayer/ext_delayer_acl.8 --- squid-4.8/src/acl/external/delayer/ext_delayer_acl.8 2019-07-10 07:25:12.000000000 +1200 +++ squid-4.9/src/acl/external/delayer/ext_delayer_acl.8 2019-11-06 08:27:23.000000000 +1300 @@ -133,7 +133,7 @@ .\" ======================================================================== .\" .IX Title "EXT_DELAYER_ACL 8" -.TH EXT_DELAYER_ACL 8 "2019-07-09" "perl v5.28.1" "User Contributed Perl Documentation" +.TH EXT_DELAYER_ACL 8 "2019-11-05" "perl v5.28.1" "User Contributed Perl Documentation" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l diff -u -r -N squid-4.8/src/acl/external/SQL_session/ext_sql_session_acl.8 squid-4.9/src/acl/external/SQL_session/ext_sql_session_acl.8 --- squid-4.8/src/acl/external/SQL_session/ext_sql_session_acl.8 2019-07-10 07:25:12.000000000 +1200 +++ squid-4.9/src/acl/external/SQL_session/ext_sql_session_acl.8 2019-11-06 08:27:24.000000000 +1300 @@ -133,7 +133,7 @@ .\" ======================================================================== .\" .IX Title "EXT_SQL_SESSION_ACL 8" -.TH EXT_SQL_SESSION_ACL 8 "2019-07-09" "perl v5.28.1" "User Contributed Perl Documentation" +.TH EXT_SQL_SESSION_ACL 8 "2019-11-05" "perl v5.28.1" "User Contributed Perl Documentation" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l diff -u -r -N squid-4.8/src/acl/external/wbinfo_group/ext_wbinfo_group_acl.8 squid-4.9/src/acl/external/wbinfo_group/ext_wbinfo_group_acl.8 --- squid-4.8/src/acl/external/wbinfo_group/ext_wbinfo_group_acl.8 2019-07-10 07:25:12.000000000 +1200 +++ squid-4.9/src/acl/external/wbinfo_group/ext_wbinfo_group_acl.8 2019-11-06 08:27:24.000000000 +1300 @@ -133,7 +133,7 @@ .\" ======================================================================== .\" .IX Title "EXT_WBINFO_GROUP_ACL 8" -.TH EXT_WBINFO_GROUP_ACL 8 "2019-07-09" "perl v5.28.1" "User Contributed Perl Documentation" +.TH EXT_WBINFO_GROUP_ACL 8 "2019-11-05" "perl v5.28.1" "User Contributed Perl Documentation" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l diff -u -r -N squid-4.8/src/adaptation/ecap/MessageRep.cc squid-4.9/src/adaptation/ecap/MessageRep.cc --- squid-4.8/src/adaptation/ecap/MessageRep.cc 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/src/adaptation/ecap/MessageRep.cc 2019-11-06 08:14:40.000000000 +1300 @@ -200,8 +200,7 @@ { // TODO: if method is not set, AnyP::Uri::parse will assume it is not connect; // Can we change AnyP::Uri::parse API to remove the method parameter? - const char *buf = aUri.toString().c_str(); - const bool ok = theMessage.url.parse(theMessage.method, buf); + const auto ok = theMessage.url.parse(theMessage.method, SBuf(aUri.toString())); Must(ok); } diff -u -r -N squid-4.8/src/anyp/ProtocolType.h squid-4.9/src/anyp/ProtocolType.h --- squid-4.8/src/anyp/ProtocolType.h 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/src/anyp/ProtocolType.h 2019-11-06 08:14:40.000000000 +1300 @@ -14,6 +14,7 @@ namespace AnyP { +// TODO order by current protocol popularity (eg HTTPS before FTP) /** * List of all protocols known and supported. * This is a combined list. It is used as type-codes where needed and diff -u -r -N squid-4.8/src/anyp/Uri.cc squid-4.9/src/anyp/Uri.cc --- squid-4.8/src/anyp/Uri.cc 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/src/anyp/Uri.cc 2019-11-06 08:14:40.000000000 +1300 @@ -12,6 +12,7 @@ #include "anyp/Uri.h" #include "globals.h" #include "HttpRequest.h" +#include "parser/Tokenizer.h" #include "rfc1738.h" #include "SquidConfig.h" #include "SquidString.h" @@ -59,6 +60,16 @@ touch(); } +SBuf +AnyP::Uri::hostOrIp() const +{ + static char ip[MAX_IPSTRLEN]; + if (hostIsNumeric()) + return SBuf(hostIP().toStr(ip, sizeof(ip))); + else + return SBuf(host()); +} + const SBuf & AnyP::Uri::path() const { @@ -116,346 +127,393 @@ } /** - * Parse the scheme name from string b, into protocol type. - * The string must be 0-terminated. + * Extract the URI scheme and ':' delimiter from the given input buffer. + * + * Schemes up to 16 characters are accepted. + * + * Governed by RFC 3986 section 3.1 */ -AnyP::ProtocolType -urlParseProtocol(const char *b) +static AnyP::UriScheme +uriParseScheme(Parser::Tokenizer &tok) { - // make e point to the ':' character - const char *e = b + strcspn(b, ":"); - int len = e - b; - - /* test common stuff first */ - - if (strncasecmp(b, "http", len) == 0) - return AnyP::PROTO_HTTP; - - if (strncasecmp(b, "ftp", len) == 0) - return AnyP::PROTO_FTP; - - if (strncasecmp(b, "https", len) == 0) - return AnyP::PROTO_HTTPS; - - if (strncasecmp(b, "file", len) == 0) - return AnyP::PROTO_FTP; - - if (strncasecmp(b, "coap", len) == 0) - return AnyP::PROTO_COAP; - - if (strncasecmp(b, "coaps", len) == 0) - return AnyP::PROTO_COAPS; - - if (strncasecmp(b, "gopher", len) == 0) - return AnyP::PROTO_GOPHER; - - if (strncasecmp(b, "wais", len) == 0) - return AnyP::PROTO_WAIS; - - if (strncasecmp(b, "cache_object", len) == 0) - return AnyP::PROTO_CACHE_OBJECT; - - if (strncasecmp(b, "urn", len) == 0) - return AnyP::PROTO_URN; - - if (strncasecmp(b, "whois", len) == 0) - return AnyP::PROTO_WHOIS; + /* + * RFC 3986 section 3.1 paragraph 2: + * + * Scheme names consist of a sequence of characters beginning with a + * letter and followed by any combination of letters, digits, plus + * ("+"), period ("."), or hyphen ("-"). + * + * The underscore ("_") required to match "cache_object://" squid + * special URI scheme. + */ + static const auto schemeChars = +#if USE_HTTP_VIOLATIONS + CharacterSet("special", "_") + +#endif + CharacterSet("scheme", "+.-") + CharacterSet::ALPHA + CharacterSet::DIGIT; - if (len > 0) - return AnyP::PROTO_UNKNOWN; + SBuf str; + if (tok.prefix(str, schemeChars, 16) && tok.skip(':') && CharacterSet::ALPHA[str.at(0)]) { + const auto protocol = AnyP::UriScheme::FindProtocolType(str); + if (protocol == AnyP::PROTO_UNKNOWN) + return AnyP::UriScheme(protocol, str.c_str()); + return AnyP::UriScheme(protocol, nullptr); + } - return AnyP::PROTO_NONE; + throw TextException("invalid URI scheme", Here()); } -/* - * Parse a URI/URL. - * - * Stores parsed values in the `request` argument. +/** + * Appends configured append_domain to hostname, assuming + * the given buffer is at least SQUIDHOSTNAMELEN bytes long, + * and that the host FQDN is not a 'dotless' TLD. * - * This abuses HttpRequest as a way of representing the parsed url - * and its components. - * method is used to switch parsers and to init the HttpRequest. - * If method is Http::METHOD_CONNECT, then rather than a URL a hostname:port is - * looked for. - * The url is non const so that if its too long we can NULL-terminate it in place. + * \returns false if and only if there is not enough space to append */ +bool +urlAppendDomain(char *host) +{ + /* For IPv4 addresses check for a dot */ + /* For IPv6 addresses also check for a colon */ + if (Config.appendDomain && !strchr(host, '.') && !strchr(host, ':')) { + const uint64_t dlen = strlen(host); + const uint64_t want = dlen + Config.appendDomainLen; + if (want > SQUIDHOSTNAMELEN - 1) { + debugs(23, 2, "URL domain too large (" << dlen << " bytes)"); + return false; + } + strncat(host, Config.appendDomain, SQUIDHOSTNAMELEN - dlen - 1); + } + return true; +} /* - * This routine parses a URL. Its assumed that the URL is complete - + * Parse a URI/URL. + * + * It is assumed that the URL is complete - * ie, the end of the string is the end of the URL. Don't pass a partial * URL here as this routine doesn't have any way of knowing whether - * its partial or not (ie, it handles the case of no trailing slash as + * it is partial or not (ie, it handles the case of no trailing slash as * being "end of host with implied path of /". + * + * method is used to switch parsers. If method is Http::METHOD_CONNECT, + * then rather than a URL a hostname:port is looked for. */ bool -AnyP::Uri::parse(const HttpRequestMethod& method, const char *url) +AnyP::Uri::parse(const HttpRequestMethod& method, const SBuf &rawUrl) { - LOCAL_ARRAY(char, proto, MAX_URL); - LOCAL_ARRAY(char, login, MAX_URL); - LOCAL_ARRAY(char, foundHost, MAX_URL); - LOCAL_ARRAY(char, urlpath, MAX_URL); - char *t = NULL; - char *q = NULL; - int foundPort; - AnyP::ProtocolType protocol = AnyP::PROTO_NONE; - int l; - int i; - const char *src; - char *dst; - proto[0] = foundHost[0] = urlpath[0] = login[0] = '\0'; - - if ((l = strlen(url)) + Config.appendDomainLen > (MAX_URL - 1)) { - debugs(23, DBG_IMPORTANT, MYNAME << "URL too large (" << l << " bytes)"); - return false; - } - if (method == Http::METHOD_CONNECT) { - /* - * RFC 7230 section 5.3.3: authority-form = authority - * "excluding any userinfo and its "@" delimiter" - * - * RFC 3986 section 3.2: authority = [ userinfo "@" ] host [ ":" port ] - * - * As an HTTP(S) proxy we assume HTTPS (443) if no port provided. - */ - foundPort = 443; + try { - if (sscanf(url, "[%[^]]]:%d", foundHost, &foundPort) < 1) - if (sscanf(url, "%[^:]:%d", foundHost, &foundPort) < 1) - return false; - - } else if ((method == Http::METHOD_OPTIONS || method == Http::METHOD_TRACE) && - AnyP::Uri::Asterisk().cmp(url) == 0) { - parseFinish(AnyP::PROTO_HTTP, nullptr, url, foundHost, SBuf(), 80 /* HTTP default port */); - return true; - } else if (strncmp(url, "urn:", 4) == 0) { - debugs(23, 3, "Split URI '" << url << "' into proto='urn', path='" << (url+4) << "'"); - debugs(50, 5, "urn=" << (url+4)); - setScheme(AnyP::PROTO_URN, nullptr); - path(url + 4); - return true; - } else { - /* Parse the URL: */ - src = url; - i = 0; - /* Find first : - everything before is protocol */ - for (i = 0, dst = proto; i < l && *src != ':'; ++i, ++src, ++dst) { - *dst = *src; - } - if (i >= l) - return false; - *dst = '\0'; + LOCAL_ARRAY(char, login, MAX_URL); + LOCAL_ARRAY(char, foundHost, MAX_URL); + LOCAL_ARRAY(char, urlpath, MAX_URL); + char *t = NULL; + char *q = NULL; + int foundPort; + int l; + int i; + const char *src; + char *dst; + foundHost[0] = urlpath[0] = login[0] = '\0'; - /* Then its :// */ - if ((i+3) > l || *src != ':' || *(src + 1) != '/' || *(src + 2) != '/') + if ((l = rawUrl.length()) + Config.appendDomainLen > (MAX_URL - 1)) { + debugs(23, DBG_IMPORTANT, MYNAME << "URL too large (" << l << " bytes)"); return false; - i += 3; - src += 3; + } - /* Then everything until first /; thats host (and port; which we'll look for here later) */ - // bug 1881: If we don't get a "/" then we imply it was there - // bug 3074: We could just be given a "?" or "#". These also imply "/" - // bug 3233: whitespace is also a hostname delimiter. - for (dst = foundHost; i < l && *src != '/' && *src != '?' && *src != '#' && *src != '\0' && !xisspace(*src); ++i, ++src, ++dst) { - *dst = *src; - } - - /* - * We can't check for "i >= l" here because we could be at the end of the line - * and have a perfectly valid URL w/ no trailing '/'. In this case we assume we've - * been -given- a valid URL and the path is just '/'. - */ - if (i > l) - return false; - *dst = '\0'; + if ((method == Http::METHOD_OPTIONS || method == Http::METHOD_TRACE) && + Asterisk().cmp(rawUrl) == 0) { + // XXX: these methods might also occur in HTTPS traffic. Handle this better. + setScheme(AnyP::PROTO_HTTP, nullptr); + port(getScheme().defaultPort()); + path(Asterisk()); + return true; + } + + Parser::Tokenizer tok(rawUrl); + AnyP::UriScheme scheme; + + if (method == Http::METHOD_CONNECT) { + /* + * RFC 7230 section 5.3.3: authority-form = authority + * "excluding any userinfo and its "@" delimiter" + * + * RFC 3986 section 3.2: authority = [ userinfo "@" ] host [ ":" port ] + * + * As an HTTP(S) proxy we assume HTTPS (443) if no port provided. + */ + foundPort = 443; + + // XXX: use tokenizer + auto B = tok.buf(); + const char *url = B.c_str(); + + if (sscanf(url, "[%[^]]]:%d", foundHost, &foundPort) < 1) + if (sscanf(url, "%[^:]:%d", foundHost, &foundPort) < 1) + return false; - // bug 3074: received 'path' starting with '?', '#', or '\0' implies '/' - if (*src == '?' || *src == '#' || *src == '\0') { - urlpath[0] = '/'; - dst = &urlpath[1]; } else { - dst = urlpath; - } - /* Then everything from / (inclusive) until \r\n or \0 - thats urlpath */ - for (; i < l && *src != '\r' && *src != '\n' && *src != '\0'; ++i, ++src, ++dst) { - *dst = *src; - } - /* We -could- be at the end of the buffer here */ - if (i > l) - return false; - /* If the URL path is empty we set it to be "/" */ - if (dst == urlpath) { - *dst = '/'; - ++dst; - } - *dst = '\0'; - - protocol = urlParseProtocol(proto); - foundPort = AnyP::UriScheme(protocol).defaultPort(); - - /* Is there any login information? (we should eventually parse it above) */ - t = strrchr(foundHost, '@'); - if (t != NULL) { - strncpy((char *) login, (char *) foundHost, sizeof(login)-1); - login[sizeof(login)-1] = '\0'; - t = strrchr(login, '@'); - *t = 0; - strncpy((char *) foundHost, t + 1, sizeof(foundHost)-1); - foundHost[sizeof(foundHost)-1] = '\0'; - // Bug 4498: URL-unescape the login info after extraction - rfc1738_unescape(login); - } - - /* Is there any host information? (we should eventually parse it above) */ - if (*foundHost == '[') { - /* strip any IPA brackets. valid under IPv6. */ - dst = foundHost; - /* only for IPv6 sadly, pre-IPv6/URL code can't handle the clean result properly anyway. */ - src = foundHost; - ++src; - l = strlen(foundHost); - i = 1; - for (; i < l && *src != ']' && *src != '\0'; ++i, ++src, ++dst) { + scheme = uriParseScheme(tok); + + if (scheme == AnyP::PROTO_NONE) + return false; // invalid scheme + + if (scheme == AnyP::PROTO_URN) { + parseUrn(tok); // throws on any error + return true; + } + + // URLs then have "//" + static const SBuf doubleSlash("//"); + if (!tok.skip(doubleSlash)) + return false; + + auto B = tok.remaining(); + const char *url = B.c_str(); + + /* Parse the URL: */ + src = url; + i = 0; + + /* Then everything until first /; thats host (and port; which we'll look for here later) */ + // bug 1881: If we don't get a "/" then we imply it was there + // bug 3074: We could just be given a "?" or "#". These also imply "/" + // bug 3233: whitespace is also a hostname delimiter. + for (dst = foundHost; i < l && *src != '/' && *src != '?' && *src != '#' && *src != '\0' && !xisspace(*src); ++i, ++src, ++dst) { *dst = *src; } - /* we moved in-place, so truncate the actual hostname found */ + /* + * We can't check for "i >= l" here because we could be at the end of the line + * and have a perfectly valid URL w/ no trailing '/'. In this case we assume we've + * been -given- a valid URL and the path is just '/'. + */ + if (i > l) + return false; *dst = '\0'; - ++dst; - /* skip ahead to either start of port, or original EOS */ - while (*dst != '\0' && *dst != ':') + // bug 3074: received 'path' starting with '?', '#', or '\0' implies '/' + if (*src == '?' || *src == '#' || *src == '\0') { + urlpath[0] = '/'; + dst = &urlpath[1]; + } else { + dst = urlpath; + } + /* Then everything from / (inclusive) until \r\n or \0 - thats urlpath */ + for (; i < l && *src != '\r' && *src != '\n' && *src != '\0'; ++i, ++src, ++dst) { + *dst = *src; + } + + /* We -could- be at the end of the buffer here */ + if (i > l) + return false; + /* If the URL path is empty we set it to be "/" */ + if (dst == urlpath) { + *dst = '/'; ++dst; - t = dst; - } else { - t = strrchr(foundHost, ':'); + } + *dst = '\0'; - if (t != strchr(foundHost,':') ) { - /* RFC 2732 states IPv6 "SHOULD" be bracketed. allowing for times when its not. */ - /* RFC 3986 'update' simply modifies this to an "is" with no emphasis at all! */ - /* therefore we MUST accept the case where they are not bracketed at all. */ - t = NULL; + foundPort = scheme.defaultPort(); // may be reset later + + /* Is there any login information? (we should eventually parse it above) */ + t = strrchr(foundHost, '@'); + if (t != NULL) { + strncpy((char *) login, (char *) foundHost, sizeof(login)-1); + login[sizeof(login)-1] = '\0'; + t = strrchr(login, '@'); + *t = 0; + strncpy((char *) foundHost, t + 1, sizeof(foundHost)-1); + foundHost[sizeof(foundHost)-1] = '\0'; + // Bug 4498: URL-unescape the login info after extraction + rfc1738_unescape(login); } - } - // Bug 3183 sanity check: If scheme is present, host must be too. - if (protocol != AnyP::PROTO_NONE && foundHost[0] == '\0') { - debugs(23, DBG_IMPORTANT, "SECURITY ALERT: Missing hostname in URL '" << url << "'. see access.log for details."); - return false; - } + /* Is there any host information? (we should eventually parse it above) */ + if (*foundHost == '[') { + /* strip any IPA brackets. valid under IPv6. */ + dst = foundHost; + /* only for IPv6 sadly, pre-IPv6/URL code can't handle the clean result properly anyway. */ + src = foundHost; + ++src; + l = strlen(foundHost); + i = 1; + for (; i < l && *src != ']' && *src != '\0'; ++i, ++src, ++dst) { + *dst = *src; + } + + /* we moved in-place, so truncate the actual hostname found */ + *dst = '\0'; + ++dst; + + /* skip ahead to either start of port, or original EOS */ + while (*dst != '\0' && *dst != ':') + ++dst; + t = dst; + } else { + t = strrchr(foundHost, ':'); + + if (t != strchr(foundHost,':') ) { + /* RFC 2732 states IPv6 "SHOULD" be bracketed. allowing for times when its not. */ + /* RFC 3986 'update' simply modifies this to an "is" with no emphasis at all! */ + /* therefore we MUST accept the case where they are not bracketed at all. */ + t = NULL; + } + } + + // Bug 3183 sanity check: If scheme is present, host must be too. + if (scheme != AnyP::PROTO_NONE && foundHost[0] == '\0') { + debugs(23, DBG_IMPORTANT, "SECURITY ALERT: Missing hostname in URL '" << url << "'. see access.log for details."); + return false; + } - if (t && *t == ':') { - *t = '\0'; - ++t; - foundPort = atoi(t); + if (t && *t == ':') { + *t = '\0'; + ++t; + foundPort = atoi(t); + } } - } - for (t = foundHost; *t; ++t) - *t = xtolower(*t); + for (t = foundHost; *t; ++t) + *t = xtolower(*t); - if (stringHasWhitespace(foundHost)) { - if (URI_WHITESPACE_STRIP == Config.uri_whitespace) { - t = q = foundHost; - while (*t) { - if (!xisspace(*t)) { - *q = *t; - ++q; + if (stringHasWhitespace(foundHost)) { + if (URI_WHITESPACE_STRIP == Config.uri_whitespace) { + t = q = foundHost; + while (*t) { + if (!xisspace(*t)) { + *q = *t; + ++q; + } + ++t; } - ++t; + *q = '\0'; } - *q = '\0'; } - } - debugs(23, 3, "Split URL '" << url << "' into proto='" << proto << "', host='" << foundHost << "', port='" << foundPort << "', path='" << urlpath << "'"); + debugs(23, 3, "Split URL '" << rawUrl << "' into proto='" << scheme.image() << "', host='" << foundHost << "', port='" << foundPort << "', path='" << urlpath << "'"); - if (Config.onoff.check_hostnames && - strspn(foundHost, Config.onoff.allow_underscore ? valid_hostname_chars_u : valid_hostname_chars) != strlen(foundHost)) { - debugs(23, DBG_IMPORTANT, MYNAME << "Illegal character in hostname '" << foundHost << "'"); - return false; - } + if (Config.onoff.check_hostnames && + strspn(foundHost, Config.onoff.allow_underscore ? valid_hostname_chars_u : valid_hostname_chars) != strlen(foundHost)) { + debugs(23, DBG_IMPORTANT, MYNAME << "Illegal character in hostname '" << foundHost << "'"); + return false; + } - /* For IPV6 addresses also check for a colon */ - if (Config.appendDomain && !strchr(foundHost, '.') && !strchr(foundHost, ':')) - strncat(foundHost, Config.appendDomain, SQUIDHOSTNAMELEN - strlen(foundHost) - 1); - - /* remove trailing dots from hostnames */ - while ((l = strlen(foundHost)) > 0 && foundHost[--l] == '.') - foundHost[l] = '\0'; - - /* reject duplicate or leading dots */ - if (strstr(foundHost, "..") || *foundHost == '.') { - debugs(23, DBG_IMPORTANT, MYNAME << "Illegal hostname '" << foundHost << "'"); - return false; - } + if (!urlAppendDomain(foundHost)) + return false; - if (foundPort < 1 || foundPort > 65535) { - debugs(23, 3, "Invalid port '" << foundPort << "'"); - return false; - } + /* remove trailing dots from hostnames */ + while ((l = strlen(foundHost)) > 0 && foundHost[--l] == '.') + foundHost[l] = '\0'; + + /* reject duplicate or leading dots */ + if (strstr(foundHost, "..") || *foundHost == '.') { + debugs(23, DBG_IMPORTANT, MYNAME << "Illegal hostname '" << foundHost << "'"); + return false; + } + + if (foundPort < 1 || foundPort > 65535) { + debugs(23, 3, "Invalid port '" << foundPort << "'"); + return false; + } #if HARDCODE_DENY_PORTS - /* These ports are filtered in the default squid.conf, but - * maybe someone wants them hardcoded... */ - if (foundPort == 7 || foundPort == 9 || foundPort == 19) { - debugs(23, DBG_CRITICAL, MYNAME << "Deny access to port " << foundPort); - return false; - } + /* These ports are filtered in the default squid.conf, but + * maybe someone wants them hardcoded... */ + if (foundPort == 7 || foundPort == 9 || foundPort == 19) { + debugs(23, DBG_CRITICAL, MYNAME << "Deny access to port " << foundPort); + return false; + } #endif - if (stringHasWhitespace(urlpath)) { - debugs(23, 2, "URI has whitespace: {" << url << "}"); + if (stringHasWhitespace(urlpath)) { + debugs(23, 2, "URI has whitespace: {" << rawUrl << "}"); - switch (Config.uri_whitespace) { + switch (Config.uri_whitespace) { - case URI_WHITESPACE_DENY: - return false; + case URI_WHITESPACE_DENY: + return false; - case URI_WHITESPACE_ALLOW: - break; + case URI_WHITESPACE_ALLOW: + break; - case URI_WHITESPACE_ENCODE: - t = rfc1738_escape_unescaped(urlpath); - xstrncpy(urlpath, t, MAX_URL); - break; - - case URI_WHITESPACE_CHOP: - *(urlpath + strcspn(urlpath, w_space)) = '\0'; - break; - - case URI_WHITESPACE_STRIP: - default: - t = q = urlpath; - while (*t) { - if (!xisspace(*t)) { - *q = *t; - ++q; + case URI_WHITESPACE_ENCODE: + t = rfc1738_escape_unescaped(urlpath); + xstrncpy(urlpath, t, MAX_URL); + break; + + case URI_WHITESPACE_CHOP: + *(urlpath + strcspn(urlpath, w_space)) = '\0'; + break; + + case URI_WHITESPACE_STRIP: + default: + t = q = urlpath; + while (*t) { + if (!xisspace(*t)) { + *q = *t; + ++q; + } + ++t; } - ++t; + *q = '\0'; } - *q = '\0'; } - } - parseFinish(protocol, proto, urlpath, foundHost, SBuf(login), foundPort); - return true; + setScheme(scheme); + path(urlpath); + host(foundHost); + userInfo(SBuf(login)); + port(foundPort); + return true; + + } catch (...) { + debugs(23, 2, "error: " << CurrentException << " " << Raw("rawUrl", rawUrl.rawContent(), rawUrl.length())); + return false; + } } -/// Update the URL object with parsed URI data. +/** + * Governed by RFC 8141 section 2: + * + * assigned-name = "urn" ":" NID ":" NSS + * NID = (alphanum) 0*30(ldh) (alphanum) + * ldh = alphanum / "-" + * NSS = pchar *(pchar / "/") + * + * RFC 3986 Appendix D.2 defines (as deprecated): + * + * alphanum = ALPHA / DIGIT + * + * Notice that NID is exactly 2-32 characters in length. + */ void -AnyP::Uri::parseFinish(const AnyP::ProtocolType protocol, - const char *const protoStr, // for unknown protocols - const char *const aUrlPath, - const char *const aHost, - const SBuf &aLogin, - const int aPort) -{ - setScheme(protocol, protoStr); - path(aUrlPath); - host(aHost); - userInfo(aLogin); - port(aPort); +AnyP::Uri::parseUrn(Parser::Tokenizer &tok) +{ + static const auto nidChars = CharacterSet("NID","-") + CharacterSet::ALPHA + CharacterSet::DIGIT; + static const auto alphanum = (CharacterSet::ALPHA + CharacterSet::DIGIT).rename("alphanum"); + SBuf nid; + if (!tok.prefix(nid, nidChars, 32)) + throw TextException("NID not found", Here()); + + if (!tok.skip(':')) + throw TextException("NID too long or missing ':' delimiter", Here()); + + if (nid.length() < 2) + throw TextException("NID too short", Here()); + + if (!alphanum[*nid.begin()]) + throw TextException("NID prefix is not alphanumeric", Here()); + + if (!alphanum[*nid.rbegin()]) + throw TextException("NID suffix is not alphanumeric", Here()); + + setScheme(AnyP::PROTO_URN, nullptr); + host(nid.c_str()); + // TODO validate path characters + path(tok.remaining()); + debugs(23, 3, "Split URI into proto=urn, nid=" << nid << ", " << Raw("path",path().rawContent(),path().length())); } void @@ -503,6 +561,9 @@ absolute_.append("@", 1); } absolute_.append(authority()); + } else { + absolute_.append(host()); + absolute_.append(":", 1); } absolute_.append(path()); } diff -u -r -N squid-4.8/src/anyp/Uri.h squid-4.9/src/anyp/Uri.h --- squid-4.8/src/anyp/Uri.h 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/src/anyp/Uri.h 2019-11-06 08:14:40.000000000 +1300 @@ -11,6 +11,7 @@ #include "anyp/UriScheme.h" #include "ip/Address.h" +#include "parser/Tokenizer.h" #include "rfc2181.h" #include "sbuf/SBuf.h" @@ -59,7 +60,7 @@ } void touch(); ///< clear the cached URI display forms - bool parse(const HttpRequestMethod &, const char *url); + bool parse(const HttpRequestMethod &, const SBuf &url); /// \return a new URI that honors uri_whitespace static char *cleanup(const char *uri); @@ -71,6 +72,10 @@ scheme_ = AnyP::UriScheme(p, str); touch(); } + void setScheme(const AnyP::UriScheme &s) { + scheme_ = s; + touch(); + } void userInfo(const SBuf &s) {userInfo_=s; touch();} const SBuf &userInfo() const {return userInfo_;} @@ -80,6 +85,11 @@ int hostIsNumeric(void) const {return hostIsNumeric_;} Ip::Address const & hostIP(void) const {return hostAddr_;} + /// \returns the host subcomponent of the authority component + /// If the host is an IPv6 address, returns that IP address without + /// [brackets]! See RFC 3986 Section 3.2.2. + SBuf hostOrIp() const; + void port(unsigned short p) {port_=p; touch();} unsigned short port() const {return port_;} @@ -115,7 +125,7 @@ SBuf &absolute() const; private: - void parseFinish(const AnyP::ProtocolType, const char *const, const char *const, const char *const, const SBuf &, const int); + void parseUrn(Parser::Tokenizer&); /** \par @@ -191,6 +201,7 @@ char *urlMakeAbsolute(const HttpRequest *, const char *); char *urlRInternal(const char *host, unsigned short port, const char *dir, const char *name); char *urlInternal(const char *dir, const char *name); +bool urlAppendDomain(char *host); ///< apply append_domain config to the given hostname enum MatchDomainNameFlags { mdnNone = 0, diff -u -r -N squid-4.8/src/anyp/UriScheme.cc squid-4.9/src/anyp/UriScheme.cc --- squid-4.8/src/anyp/UriScheme.cc 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/src/anyp/UriScheme.cc 2019-11-06 08:14:40.000000000 +1300 @@ -48,6 +48,25 @@ } } +const AnyP::ProtocolType +AnyP::UriScheme::FindProtocolType(const SBuf &scheme) +{ + if (scheme.isEmpty()) + return AnyP::PROTO_NONE; + + Init(); + + auto img = scheme; + img.toLower(); + // TODO: use base/EnumIterator.h if possible + for (int i = AnyP::PROTO_NONE + 1; i < AnyP::PROTO_UNKNOWN; ++i) { + if (LowercaseSchemeNames_.at(i) == img) + return AnyP::ProtocolType(i); + } + + return AnyP::PROTO_UNKNOWN; +} + unsigned short AnyP::UriScheme::defaultPort() const { diff -u -r -N squid-4.8/src/anyp/UriScheme.h squid-4.9/src/anyp/UriScheme.h --- squid-4.8/src/anyp/UriScheme.h 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/src/anyp/UriScheme.h 2019-11-06 08:14:40.000000000 +1300 @@ -54,6 +54,9 @@ /// initializes down-cased protocol scheme names array static void Init(); + /// \returns ProtocolType for the given scheme name or PROTO_UNKNOWN + static const AnyP::ProtocolType FindProtocolType(const SBuf &); + private: /// optimization: stores down-cased protocol scheme names, copied from /// AnyP::ProtocolType_str diff -u -r -N squid-4.8/src/auth/basic/DB/basic_db_auth.8 squid-4.9/src/auth/basic/DB/basic_db_auth.8 --- squid-4.8/src/auth/basic/DB/basic_db_auth.8 2019-07-10 07:25:13.000000000 +1200 +++ squid-4.9/src/auth/basic/DB/basic_db_auth.8 2019-11-06 08:27:24.000000000 +1300 @@ -133,7 +133,7 @@ .\" ======================================================================== .\" .IX Title "BASIC_DB_AUTH 8" -.TH BASIC_DB_AUTH 8 "2019-07-09" "perl v5.28.1" "User Contributed Perl Documentation" +.TH BASIC_DB_AUTH 8 "2019-11-05" "perl v5.28.1" "User Contributed Perl Documentation" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l diff -u -r -N squid-4.8/src/auth/basic/POP3/basic_pop3_auth.8 squid-4.9/src/auth/basic/POP3/basic_pop3_auth.8 --- squid-4.8/src/auth/basic/POP3/basic_pop3_auth.8 2019-07-10 07:25:13.000000000 +1200 +++ squid-4.9/src/auth/basic/POP3/basic_pop3_auth.8 2019-11-06 08:27:25.000000000 +1300 @@ -133,7 +133,7 @@ .\" ======================================================================== .\" .IX Title "BASIC_POP3_AUTH 8" -.TH BASIC_POP3_AUTH 8 "2019-07-09" "perl v5.28.1" "User Contributed Perl Documentation" +.TH BASIC_POP3_AUTH 8 "2019-11-05" "perl v5.28.1" "User Contributed Perl Documentation" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l diff -u -r -N squid-4.8/src/auth/digest/Config.cc squid-4.9/src/auth/digest/Config.cc --- squid-4.8/src/auth/digest/Config.cc 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/src/auth/digest/Config.cc 2019-11-06 08:14:40.000000000 +1300 @@ -21,13 +21,13 @@ #include "auth/Gadgets.h" #include "auth/State.h" #include "base/LookupTable.h" -#include "base64.h" #include "cache_cf.h" #include "event.h" #include "helper.h" #include "HttpHeaderTools.h" #include "HttpReply.h" #include "HttpRequest.h" +#include "md5.h" #include "mgr/Registration.h" #include "rfc2617.h" #include "sbuf/SBuf.h" @@ -89,7 +89,7 @@ */ static void authenticateDigestNonceCacheCleanup(void *data); -static digest_nonce_h *authenticateDigestNonceFindNonce(const char *nonceb64); +static digest_nonce_h *authenticateDigestNonceFindNonce(const char *noncehex); static void authenticateDigestNonceDelete(digest_nonce_h * nonce); static void authenticateDigestNonceSetup(void); static void authDigestNonceEncode(digest_nonce_h * nonce); @@ -108,11 +108,14 @@ if (nonce->key) xfree(nonce->key); - nonce->key = xcalloc(base64_encode_len(sizeof(digest_nonce_data)), 1); - struct base64_encode_ctx ctx; - base64_encode_init(&ctx); - size_t blen = base64_encode_update(&ctx, reinterpret_cast(nonce->key), sizeof(digest_nonce_data), reinterpret_cast(&(nonce->noncedata))); - blen += base64_encode_final(&ctx, reinterpret_cast(nonce->key)+blen); + SquidMD5_CTX Md5Ctx; + HASH H; + SquidMD5Init(&Md5Ctx); + SquidMD5Update(&Md5Ctx, reinterpret_cast(&nonce->noncedata), sizeof(nonce->noncedata)); + SquidMD5Final(reinterpret_cast(H), &Md5Ctx); + + nonce->key = xcalloc(sizeof(HASHHEX), 1); + CvtHex(H, static_cast(nonce->key)); } digest_nonce_h * @@ -147,12 +150,12 @@ * * Now for my reasoning: * We will not accept a unrecognised nonce->we have all recognisable - * nonces stored. If we send out unique base64 encodings we guarantee + * nonces stored. If we send out unique encodings we guarantee * that a given nonce applies to only one user (barring attacks or * really bad timing with expiry and creation). Using a random * component in the nonce allows us to loop to find a unique nonce. * We use H(nonce_data) so the nonce is meaningless to the reciever. - * So our nonce looks like base64(H(timestamp,pointertohash,randomdata)) + * So our nonce looks like hex(H(timestamp,pointertohash,randomdata)) * And even if our randomness is not very random we don't really care * - the timestamp and memory pointer also guarantee local uniqueness * in the input to the hash function. @@ -251,7 +254,7 @@ authenticateDigestNonceCacheCleanup(void *) { /* - * We walk the hash by nonceb64 as that is the unique key we + * We walk the hash by noncehex as that is the unique key we * use. For big hash tables we could consider stepping through * the cache, 100/200 entries at a time. Lets see how it flies * first. @@ -320,7 +323,7 @@ } const char * -authenticateDigestNonceNonceb64(const digest_nonce_h * nonce) +authenticateDigestNonceNonceHex(const digest_nonce_h * nonce) { if (!nonce) return NULL; @@ -329,18 +332,18 @@ } static digest_nonce_h * -authenticateDigestNonceFindNonce(const char *nonceb64) +authenticateDigestNonceFindNonce(const char *noncehex) { digest_nonce_h *nonce = NULL; - if (nonceb64 == NULL) + if (noncehex == NULL) return NULL; - debugs(29, 9, "looking for nonceb64 '" << nonceb64 << "' in the nonce cache."); + debugs(29, 9, "looking for noncehex '" << noncehex << "' in the nonce cache."); - nonce = static_cast < digest_nonce_h * >(hash_lookup(digest_nonce_cache, nonceb64)); + nonce = static_cast < digest_nonce_h * >(hash_lookup(digest_nonce_cache, noncehex)); - if ((nonce == NULL) || (strcmp(authenticateDigestNonceNonceb64(nonce), nonceb64))) + if ((nonce == NULL) || (strcmp(authenticateDigestNonceNonceHex(nonce), noncehex))) return NULL; debugs(29, 9, "Found nonce '" << nonce << "'"); @@ -535,12 +538,12 @@ debugs(29, 9, "Sending type:" << hdrType << " header: 'Digest realm=\"" << realm << "\", nonce=\"" << - authenticateDigestNonceNonceb64(nonce) << "\", qop=\"" << QOP_AUTH << + authenticateDigestNonceNonceHex(nonce) << "\", qop=\"" << QOP_AUTH << "\", stale=" << (stale ? "true" : "false")); /* in the future, for WWW auth we may want to support the domain entry */ httpHeaderPutStrf(&rep->header, hdrType, "Digest realm=\"" SQUIDSBUFPH "\", nonce=\"%s\", qop=\"%s\", stale=%s", - SQUIDSBUFPRINT(realm), authenticateDigestNonceNonceb64(nonce), QOP_AUTH, stale ? "true" : "false"); + SQUIDSBUFPRINT(realm), authenticateDigestNonceNonceHex(nonce), QOP_AUTH, stale ? "true" : "false"); } /* Initialize helpers and the like for this auth scheme. Called AFTER parsing the @@ -852,10 +855,10 @@ break; case DIGEST_NONCE: - safe_free(digest_request->nonceb64); + safe_free(digest_request->noncehex); if (value.size() != 0) - digest_request->nonceb64 = xstrndup(value.rawBuf(), value.size() + 1); - debugs(29, 9, "Found nonce '" << digest_request->nonceb64 << "'"); + digest_request->noncehex = xstrndup(value.rawBuf(), value.size() + 1); + debugs(29, 9, "Found nonce '" << digest_request->noncehex << "'"); break; case DIGEST_NC: @@ -931,7 +934,7 @@ } /* and a nonce? */ - if (!digest_request->nonceb64 || digest_request->nonceb64[0] == '\0') { + if (!digest_request->noncehex || digest_request->noncehex[0] == '\0') { debugs(29, 2, "Empty or not present nonce"); rv = authDigestLogUsername(username, digest_request, aRequestRealm); safe_free(username); @@ -1006,7 +1009,7 @@ /** below nonce state dependent **/ /* now the nonce */ - nonce = authenticateDigestNonceFindNonce(digest_request->nonceb64); + nonce = authenticateDigestNonceFindNonce(digest_request->noncehex); /* check that we're not being hacked / the username hasn't changed */ if (nonce && nonce->user && strcmp(username, nonce->user->username())) { debugs(29, 2, "Username for the nonce does not equal the username for the request"); @@ -1082,7 +1085,7 @@ debugs(29, 9, "username = '" << digest_user->username() << "'\nrealm = '" << digest_request->realm << "'\nqop = '" << digest_request->qop << "'\nalgorithm = '" << digest_request->algorithm << "'\nuri = '" << - digest_request->uri << "'\nnonce = '" << digest_request->nonceb64 << + digest_request->uri << "'\nnonce = '" << digest_request->noncehex << "'\nnc = '" << digest_request->nc << "'\ncnonce = '" << digest_request->cnonce << "'\nresponse = '" << digest_request->response << "'\ndigestnonce = '" << nonce << "'"); diff -u -r -N squid-4.8/src/auth/digest/Config.h squid-4.9/src/auth/digest/Config.h --- squid-4.8/src/auth/digest/Config.h 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/src/auth/digest/Config.h 2019-11-06 08:14:40.000000000 +1300 @@ -29,7 +29,7 @@ typedef struct _digest_nonce_data digest_nonce_data; typedef struct _digest_nonce_h digest_nonce_h; -/* data to be encoded into the nonce's b64 representation */ +/* data to be encoded into the nonce's hex representation */ struct _digest_nonce_data { time_t creationtime; /* in memory address of the nonce struct (similar purpose to an ETag) */ @@ -58,7 +58,7 @@ void authDigestNonceUnlink(digest_nonce_h * nonce); int authDigestNonceIsValid(digest_nonce_h * nonce, char nc[9]); int authDigestNonceIsStale(digest_nonce_h * nonce); -const char *authenticateDigestNonceNonceb64(const digest_nonce_h * nonce); +const char *authenticateDigestNonceNonceHex(const digest_nonce_h * nonce); int authDigestNonceLastRequest(digest_nonce_h * nonce); void authenticateDigestNonceShutdown(void); void authDigestNoncePurge(digest_nonce_h * nonce); diff -u -r -N squid-4.8/src/auth/digest/UserRequest.cc squid-4.9/src/auth/digest/UserRequest.cc --- squid-4.8/src/auth/digest/UserRequest.cc 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/src/auth/digest/UserRequest.cc 2019-11-06 08:14:40.000000000 +1300 @@ -23,7 +23,7 @@ #include "SquidTime.h" Auth::Digest::UserRequest::UserRequest() : - nonceb64(NULL), + noncehex(NULL), cnonce(NULL), realm(NULL), pszPass(NULL), @@ -46,7 +46,7 @@ { assert(LockCount()==0); - safe_free(nonceb64); + safe_free(noncehex); safe_free(cnonce); safe_free(realm); safe_free(pszPass); @@ -109,11 +109,11 @@ } DigestCalcHA1(digest_request->algorithm, NULL, NULL, NULL, - authenticateDigestNonceNonceb64(digest_request->nonce), + authenticateDigestNonceNonceHex(digest_request->nonce), digest_request->cnonce, digest_user->HA1, SESSIONKEY); SBuf sTmp = request->method.image(); - DigestCalcResponse(SESSIONKEY, authenticateDigestNonceNonceb64(digest_request->nonce), + DigestCalcResponse(SESSIONKEY, authenticateDigestNonceNonceHex(digest_request->nonce), digest_request->nc, digest_request->cnonce, digest_request->qop, sTmp.c_str(), digest_request->uri, HA2, Response); @@ -135,7 +135,7 @@ * used. */ sTmp = HttpRequestMethod(Http::METHOD_GET).image(); - DigestCalcResponse(SESSIONKEY, authenticateDigestNonceNonceb64(digest_request->nonce), + DigestCalcResponse(SESSIONKEY, authenticateDigestNonceNonceHex(digest_request->nonce), digest_request->nc, digest_request->cnonce, digest_request->qop, sTmp.c_str(), digest_request->uri, HA2, Response); @@ -176,7 +176,7 @@ /* check Auth::Pending to avoid loop */ if (!authDigestNonceIsValid(digest_request->nonce, digest_request->nc) && user()->credentials() != Auth::Pending) { - debugs(29, 3, auth_user->username() << "' validated OK but nonce stale: " << digest_request->nonceb64); + debugs(29, 3, auth_user->username() << "' validated OK but nonce stale: " << digest_request->noncehex); /* Pending prevent banner and makes a ldap control */ auth_user->credentials(Auth::Pending); nonce->flags.valid = false; @@ -244,8 +244,8 @@ nextnonce = authenticateDigestNonceNew(); authDigestUserLinkNonce(digest_user, nextnonce); } - debugs(29, 9, "Sending type:" << type << " header: 'nextnonce=\"" << authenticateDigestNonceNonceb64(nextnonce) << "\""); - httpHeaderPutStrf(&rep->header, type, "nextnonce=\"%s\"", authenticateDigestNonceNonceb64(nextnonce)); + debugs(29, 9, "Sending type:" << type << " header: 'nextnonce=\"" << authenticateDigestNonceNonceHex(nextnonce) << "\""); + httpHeaderPutStrf(&rep->header, type, "nextnonce=\"%s\"", authenticateDigestNonceNonceHex(nextnonce)); } } @@ -276,8 +276,8 @@ nonce = authenticateDigestNonceNew(); authDigestUserLinkNonce(digest_user, nonce); } - debugs(29, 9, "Sending type:" << type << " header: 'nextnonce=\"" << authenticateDigestNonceNonceb64(nonce) << "\""); - httpTrailerPutStrf(&rep->header, type, "nextnonce=\"%s\"", authenticateDigestNonceNonceb64(nonce)); + debugs(29, 9, "Sending type:" << type << " header: 'nextnonce=\"" << authenticateDigestNonceNonceHex(nonce) << "\""); + httpTrailerPutStrf(&rep->header, type, "nextnonce=\"%s\"", authenticateDigestNonceNonceHex(nonce)); } } #endif diff -u -r -N squid-4.8/src/auth/digest/UserRequest.h squid-4.9/src/auth/digest/UserRequest.h --- squid-4.8/src/auth/digest/UserRequest.h 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/src/auth/digest/UserRequest.h 2019-11-06 08:14:40.000000000 +1300 @@ -44,7 +44,7 @@ virtual void startHelperLookup(HttpRequest *request, AccessLogEntry::Pointer &al, AUTHCB *, void *); virtual const char *credentialsStr(); - char *nonceb64; /* "dcd98b7102dd2f0e8b11d0f600bfb0c093" */ + char *noncehex; /* "dcd98b7102dd2f0e8b11d0f600bfb0c093" */ char *cnonce; /* "0a4f113b" */ char *realm; /* = "testrealm@host.com" */ char *pszPass; /* = "Circle Of Life" */ diff -u -r -N squid-4.8/src/base/CharacterSet.cc squid-4.9/src/base/CharacterSet.cc --- squid-4.8/src/base/CharacterSet.cc 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/src/base/CharacterSet.cc 2019-11-06 08:14:40.000000000 +1300 @@ -7,7 +7,7 @@ */ #include "squid.h" -#include "CharacterSet.h" +#include "base/CharacterSet.h" #include #include diff -u -r -N squid-4.8/src/cache_cf.cc squid-4.9/src/cache_cf.cc --- squid-4.8/src/cache_cf.cc 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/src/cache_cf.cc 2019-11-06 08:14:40.000000000 +1300 @@ -2081,6 +2081,7 @@ CachePeer *p = new CachePeer; p->host = xstrdup(host_str); + Tolower(p->host); p->name = xstrdup(host_str); p->type = parseNeighborType(token); diff -u -r -N squid-4.8/src/cf.data.pre squid-4.9/src/cf.data.pre --- squid-4.8/src/cf.data.pre 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/src/cf.data.pre 2019-11-06 08:14:40.000000000 +1300 @@ -6563,6 +6563,9 @@ note key value acl ... logformat myFormat ... %{key}note ... + + This clause only supports fast acl types. + See http://wiki.squid-cache.org/SquidFaq/SquidAcl for details. DOC_END NAME: relaxed_header_parser diff -u -r -N squid-4.8/src/client_side.cc squid-4.9/src/client_side.cc --- squid-4.8/src/client_side.cc 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/src/client_side.cc 2019-11-06 08:14:40.000000000 +1300 @@ -425,7 +425,7 @@ // The al->notes and request->notes must point to the same object. (void)SyncNotes(*al, *request); for (auto i = Config.notes.begin(); i != Config.notes.end(); ++i) { - if (const char *value = (*i)->match(request, al->reply, NULL)) { + if (const char *value = (*i)->match(request, al->reply, al)) { NotePairs ¬es = SyncNotes(*al, *request); notes.add((*i)->key.termedBuf(), value); debugs(33, 3, (*i)->key.termedBuf() << " " << value); @@ -1142,7 +1142,7 @@ vport = conn->clientConnection->local.port(); char *host = NULL; - if (vhost && (host = hp->getHeaderField("Host"))) { + if (vhost && (host = hp->getHostHeaderField())) { debugs(33, 5, "ACCEL VHOST REWRITE: vhost=" << host << " + vport=" << vport); char thost[256]; if (vport > 0) { @@ -1198,7 +1198,7 @@ { char *uri = nullptr; /* BUG: Squid cannot deal with '*' URLs (RFC2616 5.1.2) */ - if (const char *host = hp->getHeaderField("Host")) { + if (const char *host = hp->getHostHeaderField()) { const SBuf &scheme = AnyP::UriScheme(conn->transferProtocol.protocol).image(); const int url_sz = scheme.length() + strlen(host) + hp->requestUri().length() + 32; uri = static_cast(xcalloc(url_sz, 1)); @@ -1222,12 +1222,12 @@ #if USE_OPENSSL if (!uri) { Must(tlsConnectPort); - Must(sslConnectHostOrIp.size()); + Must(!tlsConnectHostOrIp.isEmpty()); SBuf useHost; if (!tlsClientSni().isEmpty()) useHost = tlsClientSni(); else - useHost.assign(sslConnectHostOrIp.rawBuf(), sslConnectHostOrIp.size()); + useHost = tlsConnectHostOrIp; const SBuf &scheme = AnyP::UriScheme(transferProtocol.protocol).image(); const int url_sz = scheme.length() + useHost.length() + hp->requestUri().length() + 32; @@ -1269,65 +1269,52 @@ return uri; } -/** Parse an HTTP request - * - * \note Sets result->flags.parsed_ok to 0 if failed to parse the request, - * to 1 if the request was correctly parsed. - * \param[in] csd a ConnStateData. The caller must make sure it is not null - * \param[in] hp an Http1::RequestParser - * \param[out] mehtod_p will be set as a side-effect of the parsing. - * Pointed-to value will be set to Http::METHOD_NONE in case of - * parsing failure - * \param[out] http_ver will be set as a side-effect of the parsing - * \return NULL on incomplete requests, - * a Http::Stream on success or failure. - */ Http::Stream * -parseHttpRequest(ConnStateData *csd, const Http1::RequestParserPointer &hp) +ConnStateData::parseHttpRequest(const Http1::RequestParserPointer &hp) { /* Attempt to parse the first line; this will define where the method, url, version and header begin */ { - const bool parsedOk = hp->parse(csd->inBuf); + Must(hp); + + if (preservingClientData_) + preservedClientData = inBuf; + + const bool parsedOk = hp->parse(inBuf); // sync the buffers after parsing. - csd->inBuf = hp->remaining(); + inBuf = hp->remaining(); if (hp->needsMoreData()) { debugs(33, 5, "Incomplete request, waiting for end of request line"); return NULL; } - if (csd->mayTunnelUnsupportedProto()) { - csd->preservedClientData = hp->parsed(); - csd->preservedClientData.append(csd->inBuf); - } - if (!parsedOk) { const bool tooBig = hp->parseStatusCode == Http::scRequestHeaderFieldsTooLarge || hp->parseStatusCode == Http::scUriTooLong; - auto result = csd->abortRequestParsing( + auto result = abortRequestParsing( tooBig ? "error:request-too-large" : "error:invalid-request"); // assume that remaining leftovers belong to this bad request - if (!csd->inBuf.isEmpty()) - csd->consumeInput(csd->inBuf.length()); + if (!inBuf.isEmpty()) + consumeInput(inBuf.length()); return result; } } /* We know the whole request is in parser now */ - debugs(11, 2, "HTTP Client " << csd->clientConnection); + debugs(11, 2, "HTTP Client " << clientConnection); debugs(11, 2, "HTTP Client REQUEST:\n---------\n" << hp->method() << " " << hp->requestUri() << " " << hp->messageProtocol() << "\n" << hp->mimeHeader() << "\n----------"); /* deny CONNECT via accelerated ports */ - if (hp->method() == Http::METHOD_CONNECT && csd->port != NULL && csd->port->flags.accelSurrogate) { - debugs(33, DBG_IMPORTANT, "WARNING: CONNECT method received on " << csd->transferProtocol << " Accelerator port " << csd->port->s.port()); + if (hp->method() == Http::METHOD_CONNECT && port != NULL && port->flags.accelSurrogate) { + debugs(33, DBG_IMPORTANT, "WARNING: CONNECT method received on " << transferProtocol << " Accelerator port " << port->s.port()); debugs(33, DBG_IMPORTANT, "WARNING: for request: " << hp->method() << " " << hp->requestUri() << " " << hp->messageProtocol()); hp->parseStatusCode = Http::scMethodNotAllowed; - return csd->abortRequestParsing("error:method-not-allowed"); + return abortRequestParsing("error:method-not-allowed"); } /* RFC 7540 section 11.6 registers the method PRI as HTTP/2 specific @@ -1335,16 +1322,16 @@ * If seen it signals a broken client or proxy has corrupted the traffic. */ if (hp->method() == Http::METHOD_PRI && hp->messageProtocol() < Http::ProtocolVersion(2,0)) { - debugs(33, DBG_IMPORTANT, "WARNING: PRI method received on " << csd->transferProtocol << " port " << csd->port->s.port()); + debugs(33, DBG_IMPORTANT, "WARNING: PRI method received on " << transferProtocol << " port " << port->s.port()); debugs(33, DBG_IMPORTANT, "WARNING: for request: " << hp->method() << " " << hp->requestUri() << " " << hp->messageProtocol()); hp->parseStatusCode = Http::scMethodNotAllowed; - return csd->abortRequestParsing("error:method-not-allowed"); + return abortRequestParsing("error:method-not-allowed"); } if (hp->method() == Http::METHOD_NONE) { debugs(33, DBG_IMPORTANT, "WARNING: Unsupported method: " << hp->method() << " " << hp->requestUri() << " " << hp->messageProtocol()); hp->parseStatusCode = Http::scMethodNotAllowed; - return csd->abortRequestParsing("error:unsupported-request-method"); + return abortRequestParsing("error:unsupported-request-method"); } // Process headers after request line @@ -1355,10 +1342,10 @@ ", mime header block:\n" << hp->mimeHeader() << "\n----------"); /* Ok, all headers are received */ - ClientHttpRequest *http = new ClientHttpRequest(csd); + ClientHttpRequest *http = new ClientHttpRequest(this); http->req_sz = hp->messageHeaderSize(); - Http::Stream *result = new Http::Stream(csd->clientConnection, http); + Http::Stream *result = new Http::Stream(clientConnection, http); StoreIOBuffer tempBuffer; tempBuffer.data = result->reqbuf; @@ -1372,7 +1359,7 @@ /* set url */ debugs(33,5, "Prepare absolute URL from " << - (csd->transparent()?"intercept":(csd->port->flags.accelSurrogate ? "accel":""))); + (transparent()?"intercept":(port->flags.accelSurrogate ? "accel":""))); /* Rewrite the URL in transparent or accelerator mode */ /* NP: there are several cases to traverse here: * - standard mode (forward proxy) @@ -1386,11 +1373,11 @@ * - remote interception with PROXY protocol * - remote reverse-proxy with PROXY protocol */ - if (csd->switchedToHttps()) { - http->uri = csd->prepareTlsSwitchingURL(hp); - } else if (csd->transparent()) { + if (switchedToHttps()) { + http->uri = prepareTlsSwitchingURL(hp); + } else if (transparent()) { /* intercept or transparent mode, properly working with no failures */ - http->uri = prepareTransparentURL(csd, hp); + http->uri = prepareTransparentURL(this, hp); } else if (internalCheck(hp->requestUri())) { // NP: only matches relative-URI /* internal URL mode */ @@ -1400,9 +1387,9 @@ // But have not parsed there yet!! flag for local-only handling. http->flags.internal = true; - } else if (csd->port->flags.accelSurrogate) { + } else if (port->flags.accelSurrogate) { /* accelerator mode */ - http->uri = prepareAcceleratedURL(csd, hp); + http->uri = prepareAcceleratedURL(this, hp); http->flags.accel = true; } @@ -1546,37 +1533,50 @@ } #endif // USE_OPENSSL -/** - * Check on_unsupported_protocol checklist and return true if tunnel mode selected - * or false otherwise - */ +/// ConnStateData::tunnelOnError() wrapper. Reduces code changes. TODO: Remove. bool clientTunnelOnError(ConnStateData *conn, Http::StreamPointer &context, HttpRequest::Pointer &request, const HttpRequestMethod& method, err_type requestError) { - if (conn->mayTunnelUnsupportedProto()) { - ACLFilledChecklist checklist(Config.accessList.on_unsupported_protocol, request.getRaw(), nullptr); - checklist.al = (context && context->http) ? context->http->al : nullptr; - checklist.requestErrorType = requestError; - checklist.src_addr = conn->clientConnection->remote; - checklist.my_addr = conn->clientConnection->local; - checklist.conn(conn); - ClientHttpRequest *http = context ? context->http : nullptr; - const char *log_uri = http ? http->log_uri : nullptr; - checklist.syncAle(request.getRaw(), log_uri); - allow_t answer = checklist.fastCheck(); - if (answer.allowed() && answer.kind == 1) { - debugs(33, 3, "Request will be tunneled to server"); - if (context) { - assert(conn->pipeline.front() == context); // XXX: still assumes HTTP/1 semantics - context->finished(); // Will remove from conn->pipeline queue - } - Comm::SetSelect(conn->clientConnection->fd, COMM_SELECT_READ, NULL, NULL, 0); - return conn->initiateTunneledRequest(request, Http::METHOD_NONE, "unknown-protocol", conn->preservedClientData); - } else { - debugs(33, 3, "Continue with returning the error: " << requestError); - } + assert(conn); + assert(conn->pipeline.front() == context); + return conn->tunnelOnError(method, requestError); +} + +/// initiate tunneling if possible or return false otherwise +bool +ConnStateData::tunnelOnError(const HttpRequestMethod &method, const err_type requestError) +{ + if (!Config.accessList.on_unsupported_protocol) { + debugs(33, 5, "disabled; send error: " << requestError); + return false; + } + + if (!preservingClientData_) { + debugs(33, 3, "may have forgotten client data; send error: " << requestError); + return false; } + const auto context = pipeline.front(); + const auto http = context ? context->http : nullptr; + const auto request = http ? http->request : nullptr; + + ACLFilledChecklist checklist(Config.accessList.on_unsupported_protocol, request, nullptr); + checklist.al = http ? http->al : nullptr; + checklist.requestErrorType = requestError; + checklist.src_addr = clientConnection->remote; + checklist.my_addr = clientConnection->local; + checklist.conn(this); + const char *log_uri = http ? http->log_uri : nullptr; + checklist.syncAle(request, log_uri); + auto answer = checklist.fastCheck(); + if (answer.allowed() && answer.kind == 1) { + debugs(33, 3, "Request will be tunneled to server"); + if (context) + context->finished(); // Will remove from pipeline queue + Comm::SetSelect(clientConnection->fd, COMM_SELECT_READ, NULL, NULL, 0); + return initiateTunneledRequest(request, Http::METHOD_NONE, "unknown-protocol", preservedClientData); + } + debugs(33, 3, "denied; send error: " << requestError); return false; } @@ -2100,6 +2100,11 @@ // we have been waiting for PROXY to provide client-IP // for some lookups, ie rDNS and IDENT. whenClientIpKnown(); + + // Done with PROXY protocol which has cleared preservingClientData_. + // If the next protocol supports on_unsupported_protocol, then its + // parseOneRequest() must reset preservingClientData_. + assert(!preservingClientData_); } if (Http::StreamPointer context = parseOneRequest()) { @@ -2111,6 +2116,11 @@ context->registerWithConn(); +#if USE_OPENSSL + if (switchedToHttps()) + parsedBumpedRequestCount++; +#endif + processParsedRequest(context); parsed_req = true; // XXX: do we really need to parse everything right NOW ? @@ -2322,13 +2332,10 @@ if (!Comm::IsConnOpen(io.conn)) return; - if (mayTunnelUnsupportedProto() && !receivedFirstByte_) { - Http::StreamPointer context = pipeline.front(); - Must(context && context->http); - HttpRequest::Pointer request = context->http->request; - if (clientTunnelOnError(this, context, request, HttpRequestMethod(), ERR_REQUEST_START_TIMEOUT)) - return; - } + const err_type error = receivedFirstByte_ ? ERR_REQUEST_PARSE_TIMEOUT : ERR_REQUEST_START_TIMEOUT; + if (tunnelOnError(HttpRequestMethod(), error)) + return; + /* * Just close the connection to not confuse browsers * using persistent connections. Some browsers open @@ -2363,6 +2370,7 @@ #if USE_OPENSSL switchedToHttps_(false), parsingTlsHandshake(false), + parsedBumpedRequestCount(0), tlsConnectPort(0), sslServerBump(NULL), signAlgorithm(Ssl::algSignTrusted), @@ -2424,6 +2432,8 @@ } else whenClientIpKnown(); + // requires needProxyProtocolHeader_ which is initialized above + preservingClientData_ = shouldPreserveClientData(); } void @@ -2846,18 +2856,18 @@ } if (reply.result == Helper::BrokenHelper) { - debugs(33, 5, HERE << "Certificate for " << sslConnectHostOrIp << " cannot be generated. ssl_crtd response: " << reply); + debugs(33, 5, "Certificate for " << tlsConnectHostOrIp << " cannot be generated. ssl_crtd response: " << reply); } else if (!reply.other().hasContent()) { debugs(1, DBG_IMPORTANT, HERE << "\"ssl_crtd\" helper returned reply."); } else { Ssl::CrtdMessage reply_message(Ssl::CrtdMessage::REPLY); if (reply_message.parse(reply.other().content(), reply.other().contentSize()) != Ssl::CrtdMessage::OK) { - debugs(33, 5, HERE << "Reply from ssl_crtd for " << sslConnectHostOrIp << " is incorrect"); + debugs(33, 5, "Reply from ssl_crtd for " << tlsConnectHostOrIp << " is incorrect"); } else { if (reply.result != Helper::Okay) { - debugs(33, 5, HERE << "Certificate for " << sslConnectHostOrIp << " cannot be generated. ssl_crtd response: " << reply_message.getBody()); + debugs(33, 5, "Certificate for " << tlsConnectHostOrIp << " cannot be generated. ssl_crtd response: " << reply_message.getBody()); } else { - debugs(33, 5, HERE << "Certificate for " << sslConnectHostOrIp << " was successfully recieved from ssl_crtd"); + debugs(33, 5, "Certificate for " << tlsConnectHostOrIp << " was successfully recieved from ssl_crtd"); if (sslServerBump && (sslServerBump->act.step1 == Ssl::bumpPeek || sslServerBump->act.step1 == Ssl::bumpStare)) { doPeekAndSpliceStep(); auto ssl = fd_table[clientConnection->fd].ssl.get(); @@ -2883,7 +2893,7 @@ void ConnStateData::buildSslCertGenerationParams(Ssl::CertificateProperties &certProperties) { - certProperties.commonName = sslCommonName_.isEmpty() ? sslConnectHostOrIp.termedBuf() : sslCommonName_.c_str(); + certProperties.commonName = sslCommonName_.isEmpty() ? tlsConnectHostOrIp.c_str() : sslCommonName_.c_str(); const bool connectedOk = sslServerBump && sslServerBump->connectedOk(); if (connectedOk) { @@ -2909,7 +2919,7 @@ // CONNECT request. if (ca->alg == Ssl::algSetCommonName) { if (!param) - param = sslConnectHostOrIp.termedBuf(); + param = tlsConnectHostOrIp.c_str(); certProperties.commonName = param; certProperties.setCommonName = true; } else if (ca->alg == Ssl::algSetValidAfter) @@ -3058,7 +3068,7 @@ ConnStateData::getSslContextDone(Security::ContextPointer &ctx) { if (port->secure.generateHostCertificates && !ctx) { - debugs(33, 2, "Failed to generate TLS context for " << sslConnectHostOrIp); + debugs(33, 2, "Failed to generate TLS context for " << tlsConnectHostOrIp); } // If generated ssl context = NULL, try to use static ssl context. @@ -3099,13 +3109,18 @@ { assert(!switchedToHttps_); - sslConnectHostOrIp = request->url.host(); + // Depending on receivedFirstByte_, we are at the start of either an + // established CONNECT tunnel with the client or an intercepted TCP (and + // presumably TLS) connection from the client. Expect TLS Client Hello. + const auto insideConnectTunnel = receivedFirstByte_; + debugs(33, 5, (insideConnectTunnel ? "post-CONNECT " : "raw TLS ") << clientConnection); + + tlsConnectHostOrIp = request->url.hostOrIp(); tlsConnectPort = request->url.port(); resetSslCommonName(request->url.host()); // We are going to read new request flags.readMore = true; - debugs(33, 5, HERE << "converting " << clientConnection << " to SSL"); // keep version major.minor details the same. // but we are now performing the HTTPS handshake traffic @@ -3133,6 +3148,13 @@ receivedFirstByte_ = false; // Get more data to peek at Tls parsingTlsHandshake = true; + + // If the protocol has changed, then reset preservingClientData_. + // Otherwise, its value initially set in start() is still valid/fresh. + // shouldPreserveClientData() uses parsingTlsHandshake which is reset above. + if (insideConnectTunnel) + preservingClientData_ = shouldPreserveClientData(); + readSomeData(); } @@ -3349,9 +3371,9 @@ if (Comm::IsConnOpen(pic.connection)) { notePinnedConnectionBecameIdle(pic); - debugs(33, 5, HERE << "bumped HTTPS server: " << sslConnectHostOrIp); + debugs(33, 5, "bumped HTTPS server: " << tlsConnectHostOrIp); } else - debugs(33, 5, HERE << "Error while bumping: " << sslConnectHostOrIp); + debugs(33, 5, "Error while bumping: " << tlsConnectHostOrIp); getSslContextStart(); } @@ -3367,13 +3389,20 @@ if (pinning.serverConnection != nullptr) { static char ip[MAX_IPSTRLEN]; - pinning.serverConnection->remote.toHostStr(ip, sizeof(ip)); - connectHost.assign(ip); + connectHost = pinning.serverConnection->remote.toStr(ip, sizeof(ip)); connectPort = pinning.serverConnection->remote.port(); - } else if (cause && cause->method == Http::METHOD_CONNECT) { - // We are inside a (not fully established) CONNECT request - connectHost = cause->url.host(); + } else if (cause) { + connectHost = cause->url.hostOrIp(); connectPort = cause->url.port(); +#if USE_OPENSSL + } else if (!tlsConnectHostOrIp.isEmpty()) { + connectHost = tlsConnectHostOrIp; + connectPort = tlsConnectPort; +#endif + } else if (transparent()) { + static char ip[MAX_IPSTRLEN]; + connectHost = clientConnection->local.toStr(ip, sizeof(ip)); + connectPort = clientConnection->local.port(); } else { debugs(33, 2, "Not able to compute URL, abort request tunneling for " << reason); return false; @@ -3434,7 +3463,6 @@ clientReplyStatus, newServer, clientSocketRecipient, clientSocketDetach, newClient, tempBuffer); - http->uri = SBufToCstring(useHost); stream->flags.parsed_ok = 1; // Do we need it? stream->mayUseConnection(true); @@ -3454,6 +3482,8 @@ request->method = method; request->url.host(useHost.c_str()); request->url.port(usePort); + + http->uri = SBufToCstring(request->effectiveRequestUri()); http->initRequest(request.getRaw()); request->manager(this, http->al); @@ -4142,15 +4172,35 @@ } bool -ConnStateData::mayTunnelUnsupportedProto() +ConnStateData::shouldPreserveClientData() const { - return Config.accessList.on_unsupported_protocol + // PROXY protocol bytes are meant for us and, hence, cannot be tunneled + if (needProxyProtocolHeader_) + return false; + + // If our decision here is negative, configuration changes are irrelevant. + // Otherwise, clientTunnelOnError() rechecks configuration before tunneling. + if (!Config.accessList.on_unsupported_protocol) + return false; + + // TODO: Figure out whether/how we can support FTP tunneling. + if (port->transport.protocol == AnyP::PROTO_FTP) + return false; + #if USE_OPENSSL - && - ((port->flags.isIntercepted() && port->flags.tunnelSslBumping) - || (serverBump() && pinning.serverConnection)) + if (parsingTlsHandshake) + return true; + + // the 1st HTTP request on a bumped connection + if (!parsedBumpedRequestCount && switchedToHttps()) + return true; #endif - ; + + // the 1st HTTP request on a connection to a plain intercepting port + if (!pipeline.nrequests && !port->secure.encryptTransport && transparent()) + return true; + + return false; } std::ostream & diff -u -r -N squid-4.8/src/client_side.h squid-4.9/src/client_side.h --- squid-4.8/src/client_side.h 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/src/client_side.h 2019-11-06 08:14:40.000000000 +1300 @@ -302,14 +302,19 @@ /// generates and sends to tunnel.cc a fake request with a given payload bool initiateTunneledRequest(HttpRequest::Pointer const &cause, Http::MethodType const method, const char *reason, const SBuf &payload); - /// whether tunneling of unsupported protocol is allowed for this connection - bool mayTunnelUnsupportedProto(); + /// whether we should start saving inBuf client bytes in anticipation of + /// tunneling them to the server later (on_unsupported_protocol) + bool shouldPreserveClientData() const; + + // TODO: move to the protected section when removing clientTunnelOnError() + bool tunnelOnError(const HttpRequestMethod &, const err_type); /// build a fake http request ClientHttpRequest *buildFakeRequest(Http::MethodType const method, SBuf &useHost, unsigned short usePort, const SBuf &payload); - /// client data which may need to forward as-is to server after an - /// on_unsupported_protocol tunnel decision. + /// From-client handshake bytes (including bytes at the beginning of a + /// CONNECT tunnel) which we may need to forward as-is if their syntax does + /// not match the expected TLS or HTTP protocol (on_unsupported_protocol). SBuf preservedClientData; /* Registered Runner API */ @@ -331,6 +336,15 @@ bool handleIdleClientPinnedTlsRead(); #endif + /// Parse an HTTP request + /// \note Sets result->flags.parsed_ok to 0 if failed to parse the request, + /// to 1 if the request was correctly parsed + /// \param[in] hp an Http1::RequestParser + /// \return NULL on incomplete requests, + /// a Http::Stream on success or failure. + /// TODO: Move to HttpServer. Warning: Move requires large code nonchanges! + Http::Stream *parseHttpRequest(const Http1::RequestParserPointer &); + /// parse input buffer prefix into a single transfer protocol request /// return NULL to request more header bytes (after checking any limits) /// use abortRequestParsing() to handle parsing errors w/o creating request @@ -351,6 +365,9 @@ BodyPipe::Pointer bodyPipe; ///< set when we are reading request body + /// whether preservedClientData is valid and should be kept up to date + bool preservingClientData_; + private: /* ::Server API */ virtual bool connFinishedWithConn(int size); @@ -385,15 +402,14 @@ Auth::UserRequest::Pointer auth_; #endif - /// the parser state for current HTTP/1.x input buffer processing - Http1::RequestParserPointer parser_; - #if USE_OPENSSL bool switchedToHttps_; bool parsingTlsHandshake; ///< whether we are getting/parsing TLS Hello bytes + /// The number of parsed HTTP requests headers on a bumped client connection + uint64_t parsedBumpedRequestCount; - /// The SSL server host name appears in CONNECT request or the server ip address for the intercepted requests - String sslConnectHostOrIp; ///< The SSL server host name as passed in the CONNECT request + /// The TLS server host name appears in CONNECT request or the server ip address for the intercepted requests + SBuf tlsConnectHostOrIp; ///< The TLS server host name as passed in the CONNECT request unsigned short tlsConnectPort; ///< The TLS server port number as passed in the CONNECT request SBuf sslCommonName_; ///< CN name for SSL certificate generation @@ -441,8 +457,6 @@ CSCB clientSocketRecipient; CSD clientSocketDetach; -/* TODO: Move to HttpServer. Warning: Move requires large code nonchanges! */ -Http::Stream *parseHttpRequest(ConnStateData *, const Http1::RequestParserPointer &); void clientProcessRequest(ConnStateData *, const Http1::RequestParserPointer &, Http::Stream *); void clientPostHttpsAccept(ConnStateData *); diff -u -r -N squid-4.8/src/client_side_request.cc squid-4.9/src/client_side_request.cc --- squid-4.8/src/client_side_request.cc 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/src/client_side_request.cc 2019-11-06 08:14:40.000000000 +1300 @@ -346,7 +346,8 @@ http->uri = (char *)xcalloc(url_sz, 1); strcpy(http->uri, url); // XXX: polluting http->uri before parser validation - if ((request = HttpRequest::FromUrl(http->uri, mx, method)) == NULL) { + request = HttpRequest::FromUrlXXX(http->uri, mx, method); + if (!request) { debugs(85, 5, "Invalid URL: " << http->uri); return -1; } @@ -1262,7 +1263,7 @@ // prevent broken helpers causing too much damage. If old URL == new URL skip the re-write. if (urlNote != NULL && strcmp(urlNote, http->uri)) { AnyP::Uri tmpUrl; - if (tmpUrl.parse(old_request->method, urlNote)) { + if (tmpUrl.parse(old_request->method, SBuf(urlNote))) { HttpRequest *new_request = old_request->clone(); new_request->url = tmpUrl; debugs(61, 2, "URL-rewriter diverts URL from " << old_request->effectiveRequestUri() << " to " << new_request->effectiveRequestUri()); diff -u -r -N squid-4.8/src/Downloader.cc squid-4.9/src/Downloader.cc --- squid-4.8/src/Downloader.cc 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/src/Downloader.cc 2019-11-06 08:14:40.000000000 +1300 @@ -129,7 +129,7 @@ const HttpRequestMethod method = Http::METHOD_GET; const MasterXaction::Pointer mx = new MasterXaction(initiator_); - HttpRequest *const request = HttpRequest::FromUrl(url_.c_str(), mx, method); + auto * const request = HttpRequest::FromUrl(url_, mx, method); if (!request) { debugs(33, 5, "Invalid URI: " << url_); return false; //earlyError(...) diff -u -r -N squid-4.8/src/err_type.h squid-4.9/src/err_type.h --- squid-4.8/src/err_type.h 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/src/err_type.h 2019-11-06 08:14:40.000000000 +1300 @@ -76,6 +76,7 @@ ERR_SECURE_ACCEPT_FAIL, // Rejects the SSL connection intead of error page ERR_REQUEST_START_TIMEOUT, // Aborts the connection instead of error page + ERR_REQUEST_PARSE_TIMEOUT, // Aborts the connection instead of error page /* Cache Manager GUI can install a manager index/home page */ MGR_INDEX, diff -u -r -N squid-4.8/src/fs/rock/forward.h squid-4.9/src/fs/rock/forward.h --- squid-4.8/src/fs/rock/forward.h 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/src/fs/rock/forward.h 2019-11-06 08:14:40.000000000 +1300 @@ -32,6 +32,9 @@ /// db cell number, starting with cell 0 (always occupied by the db header) typedef sfileno SlotId; +/// unique (within a given IoState object scope) I/O transaction identifier +typedef uint64_t IoXactionId; + class Rebuild; class IoState; @@ -40,6 +43,8 @@ class DbCellHeader; +class ReadRequest; + class WriteRequest; } diff -u -r -N squid-4.8/src/fs/rock/RockIoRequests.cc squid-4.9/src/fs/rock/RockIoRequests.cc --- squid-4.8/src/fs/rock/RockIoRequests.cc 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/src/fs/rock/RockIoRequests.cc 2019-11-06 08:14:40.000000000 +1300 @@ -14,19 +14,19 @@ CBDATA_NAMESPACED_CLASS_INIT(Rock, ReadRequest); CBDATA_NAMESPACED_CLASS_INIT(Rock, WriteRequest); -Rock::ReadRequest::ReadRequest(const ::ReadRequest &base, - const IoState::Pointer &anSio): +Rock::ReadRequest::ReadRequest(const ::ReadRequest &base, const IoState::Pointer &anSio, const IoXactionId anId): ::ReadRequest(base), - sio(anSio) + sio(anSio), + id(anId) { } -Rock::WriteRequest::WriteRequest(const ::WriteRequest &base, - const IoState::Pointer &anSio): +Rock::WriteRequest::WriteRequest(const ::WriteRequest &base, const IoState::Pointer &anSio, const IoXactionId anId): ::WriteRequest(base), sio(anSio), + sidPrevious(-1), sidCurrent(-1), - sidNext(-1), + id(anId), eof(false) { } diff -u -r -N squid-4.8/src/fs/rock/RockIoRequests.h squid-4.9/src/fs/rock/RockIoRequests.h --- squid-4.8/src/fs/rock/RockIoRequests.h 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/src/fs/rock/RockIoRequests.h 2019-11-06 08:14:40.000000000 +1300 @@ -11,6 +11,7 @@ #include "DiskIO/ReadRequest.h" #include "DiskIO/WriteRequest.h" +#include "fs/rock/forward.h" #include "fs/rock/RockIoState.h" class DiskFile; @@ -23,8 +24,11 @@ CBDATA_CLASS(ReadRequest); public: - ReadRequest(const ::ReadRequest &base, const IoState::Pointer &anSio); + ReadRequest(const ::ReadRequest &, const IoState::Pointer &, const IoXactionId); IoState::Pointer sio; + + /// identifies this read transaction for the requesting IoState + IoXactionId id; }; class WriteRequest: public ::WriteRequest @@ -32,14 +36,19 @@ CBDATA_CLASS(WriteRequest); public: - WriteRequest(const ::WriteRequest &base, const IoState::Pointer &anSio); + WriteRequest(const ::WriteRequest &, const IoState::Pointer &, const IoXactionId); IoState::Pointer sio; + /* We own these two reserved slots until SwapDir links them into the map. */ + + /// slot that will point to sidCurrent in the cache_dir map + SlotId sidPrevious; + /// slot being written using this write request SlotId sidCurrent; - /// allocated next slot (negative if we are writing the last slot) - SlotId sidNext; + /// identifies this write transaction for the requesting IoState + IoXactionId id; /// whether this is the last request for the entry bool eof; diff -u -r -N squid-4.8/src/fs/rock/RockIoState.cc squid-4.9/src/fs/rock/RockIoState.cc --- squid-4.8/src/fs/rock/RockIoState.cc 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/src/fs/rock/RockIoState.cc 2019-11-06 08:14:40.000000000 +1300 @@ -35,7 +35,12 @@ dir(aDir), slotSize(dir->slotSize), objOffset(0), + sidFirst(-1), + sidPrevious(-1), sidCurrent(-1), + sidNext(-1), + requestsSent(0), + repliesReceived(0), theBuf(dir->slotSize) { e = anEntry; @@ -101,7 +106,8 @@ // if we are dealing with the first read or // if the offset went backwords, start searching from the beginning if (sidCurrent < 0 || coreOff < objOffset) { - sidCurrent = readAnchor().start; + // readers do not need sidFirst but set it for consistency/triage sake + sidCurrent = sidFirst = readAnchor().start; objOffset = 0; } @@ -126,14 +132,32 @@ len = min(len, static_cast(objOffset + currentReadableSlice().size - coreOff)); const uint64_t diskOffset = dir->diskOffset(sidCurrent); - theFile->read(new ReadRequest(::ReadRequest(buf, - diskOffset + sizeof(DbCellHeader) + coreOff - objOffset, len), this)); + const auto start = diskOffset + sizeof(DbCellHeader) + coreOff - objOffset; + const auto id = ++requestsSent; + const auto request = new ReadRequest(::ReadRequest(buf, start, len), this, id); + theFile->read(request); } void +Rock::IoState::handleReadCompletion(Rock::ReadRequest &request, const int rlen, const int errFlag) +{ + if (errFlag != DISK_OK || rlen < 0) { + debugs(79, 3, errFlag << " failure for " << *e); + return callReaderBack(request.buf, -1); + } + + if (!expectedReply(request.id)) + return callReaderBack(request.buf, -1); + + debugs(79, 5, '#' << request.id << " read " << rlen << " bytes at " << offset_ << " for " << *e); + offset_ += rlen; + callReaderBack(request.buf, rlen); +} + +/// report (already sanitized/checked) I/O results to the read initiator +void Rock::IoState::callReaderBack(const char *buf, int rlen) { - debugs(79, 5, rlen << " bytes for " << *e); splicingPoint = rlen >= 0 ? sidCurrent : -1; if (splicingPoint < 0) staleSplicingPointNext = -1; @@ -181,38 +205,29 @@ { debugs(79, 7, swap_filen << " writes " << size << " more"); - // either this is the first write or append; we do not support write gaps + // either this is the first write or append; + // we do not support write gaps or rewrites assert(!coreOff || coreOff == -1); // throw if an accepted unknown-size entry grew too big or max-size changed Must(static_cast(offset_ + size) <= static_cast(dir->maxObjectSize())); - // allocate the first slice during the first write - if (!coreOff) { - assert(sidCurrent < 0); - sidCurrent = dir->reserveSlotForWriting(); // throws on failures - assert(sidCurrent >= 0); - writeAnchor().start = sidCurrent; - } - // buffer incoming data in slot buffer and write overflowing or final slots // quit when no data left or we stopped writing on reentrant error while (size > 0 && theFile != NULL) { - assert(sidCurrent >= 0); const size_t processed = writeToBuffer(buf, size); buf += processed; size -= processed; const bool overflow = size > 0; // We do not write a full buffer without overflow because - // we would not yet know what to set the nextSlot to. + // we do not want to risk writing a payload-free slot on EOF. if (overflow) { - const auto sidNext = dir->reserveSlotForWriting(); // throws + Must(sidNext < 0); + sidNext = dir->reserveSlotForWriting(); assert(sidNext >= 0); - writeToDisk(sidNext); - } else if (Store::Root().transientReaders(*e)) { - // write partial buffer for all remote hit readers to see - writeBufToDisk(-1, false, false); + writeToDisk(); + Must(sidNext < 0); // short sidNext lifetime simplifies code logic } } @@ -228,7 +243,7 @@ return 0; if (!theBuf.size) { - // will fill the header in writeToDisk when the next slot is known + // eventually, writeToDisk() will fill this header space theBuf.appended(sizeof(DbCellHeader)); } @@ -239,44 +254,38 @@ } /// write what was buffered during write() calls -/// negative sidNext means this is the last write request for this entry void -Rock::IoState::writeToDisk(const SlotId sidNextProposal) +Rock::IoState::writeToDisk() { assert(theFile != NULL); assert(theBuf.size >= sizeof(DbCellHeader)); - const bool lastWrite = sidNextProposal < 0; + assert((sidFirst < 0) == (sidCurrent < 0)); + if (sidFirst < 0) // this is the first disk write + sidCurrent = sidFirst = dir->reserveSlotForWriting(); + + // negative sidNext means this is the last write request for this entry + const bool lastWrite = sidNext < 0; + // here, eof means that we are writing the right-most entry slot const bool eof = lastWrite && // either not updating or the updating reader has loaded everything (touchingStoreEntry() || staleSplicingPointNext < 0); - // approve sidNextProposal unless _updating_ the last slot - const SlotId sidNext = (!touchingStoreEntry() && lastWrite) ? - staleSplicingPointNext : sidNextProposal; - debugs(79, 5, "sidNext:" << sidNextProposal << "=>" << sidNext << " eof=" << eof); + debugs(79, 5, "sidCurrent=" << sidCurrent << " sidNext=" << sidNext << " eof=" << eof); // TODO: if DiskIO module is mmap-based, we should be writing whole pages // to avoid triggering read-page;new_head+old_tail;write-page overheads - writeBufToDisk(sidNext, eof, lastWrite); - theBuf.clear(); - - sidCurrent = sidNext; -} - -/// creates and submits a request to write current slot buffer to disk -/// eof is true if and only this is the last slot -void -Rock::IoState::writeBufToDisk(const SlotId sidNext, const bool eof, const bool lastWrite) -{ - // no slots after the last/eof slot (but partial slots may have a nil next) - assert(!eof || sidNext < 0); + assert(!eof || sidNext < 0); // no slots after eof // finalize db cell header DbCellHeader header; memcpy(header.key, e->key, sizeof(header.key)); - header.firstSlot = writeAnchor().start; - header.nextSlot = sidNext; + header.firstSlot = sidFirst; + + const auto lastUpdatingWrite = lastWrite && !touchingStoreEntry(); + assert(!lastUpdatingWrite || sidNext < 0); + header.nextSlot = lastUpdatingWrite ? staleSplicingPointNext : sidNext; + header.payloadSize = theBuf.size - sizeof(DbCellHeader); header.entrySize = eof ? offset_ : 0; // storeSwapOutFileClosed sets swap_file_sz after write header.version = writeAnchor().basics.timestamp; @@ -294,21 +303,52 @@ const uint64_t diskOffset = dir->diskOffset(sidCurrent); debugs(79, 5, HERE << swap_filen << " at " << diskOffset << '+' << theBuf.size); - + const auto id = ++requestsSent; WriteRequest *const r = new WriteRequest( ::WriteRequest(static_cast(wBuf), diskOffset, theBuf.size, - memFreeBufFunc(wBufCap)), this); + memFreeBufFunc(wBufCap)), this, id); r->sidCurrent = sidCurrent; - r->sidNext = sidNext; + r->sidPrevious = sidPrevious; r->eof = lastWrite; + sidPrevious = sidCurrent; + sidCurrent = sidNext; // sidNext may be cleared/negative already + sidNext = -1; + + theBuf.clear(); + // theFile->write may call writeCompleted immediatelly theFile->write(r); } +bool +Rock::IoState::expectedReply(const IoXactionId receivedId) +{ + Must(requestsSent); // paranoid: we sent some requests + Must(receivedId); // paranoid: the request was sent by some sio + Must(receivedId <= requestsSent); // paranoid: within our range + ++repliesReceived; + const auto expectedId = repliesReceived; + if (receivedId == expectedId) + return true; + + debugs(79, 3, "no; expected reply #" << expectedId << + ", but got #" << receivedId); + return false; +} + void Rock::IoState::finishedWriting(const int errFlag) { + if (sidCurrent >= 0) { + dir->noteFreeMapSlice(sidCurrent); + sidCurrent = -1; + } + if (sidNext >= 0) { + dir->noteFreeMapSlice(sidNext); + sidNext = -1; + } + // we incremented offset_ while accumulating data in write() // we do not reset writeableAnchor_ here because we still keep the lock if (touchingStoreEntry()) @@ -320,7 +360,9 @@ Rock::IoState::close(int how) { debugs(79, 3, swap_filen << " offset: " << offset_ << " how: " << how << - " buf: " << theBuf.size << " callback: " << callback); + " leftovers: " << theBuf.size << + " after " << requestsSent << '/' << repliesReceived << + " callback: " << callback); if (!theFile) { debugs(79, 3, "I/O already canceled"); @@ -333,8 +375,17 @@ switch (how) { case wroteAll: assert(theBuf.size > 0); // we never flush last bytes on our own - writeToDisk(-1); // flush last, yet unwritten slot to disk - return; // writeCompleted() will callBack() + try { + writeToDisk(); // flush last, yet unwritten slot to disk + return; // writeCompleted() will callBack() + } + catch (...) { + debugs(79, 2, "db flush error: " << CurrentException); + // TODO: Move finishedWriting() into SwapDir::writeError(). + dir->writeError(*this); + finishedWriting(DISK_ERROR); + } + return; case writerGone: dir->writeError(*this); // abort a partially stored entry diff -u -r -N squid-4.8/src/fs/rock/RockIoState.h squid-4.9/src/fs/rock/RockIoState.h --- squid-4.8/src/fs/rock/RockIoState.h 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/src/fs/rock/RockIoState.h 2019-11-06 08:14:40.000000000 +1300 @@ -9,6 +9,7 @@ #ifndef SQUID_FS_ROCK_IO_STATE_H #define SQUID_FS_ROCK_IO_STATE_H +#include "fs/rock/forward.h" #include "fs/rock/RockSwapDir.h" #include "sbuf/MemBlob.h" @@ -41,12 +42,16 @@ /// whether we are still waiting for the I/O results (i.e., not closed) bool stillWaiting() const { return theFile != NULL; } - /// forwards read data to the reader that initiated this I/O - void callReaderBack(const char *buf, int rlen); + /// forwards read data (or an error) to the reader that initiated this I/O + void handleReadCompletion(Rock::ReadRequest &request, const int rlen, const int errFlag); /// called by SwapDir::writeCompleted() after the last write and on error void finishedWriting(const int errFlag); + /// notes that the disker has satisfied the given I/O request + /// \returns whether all earlier I/O requests have been satisfied already + bool expectedReply(const IoXactionId receivedId); + /* one and only one of these will be set and locked; access via *Anchor() */ const Ipc::StoreMapAnchor *readableAnchor_; ///< starting point for reading Ipc::StoreMapAnchor *writeableAnchor_; ///< starting point for writing @@ -64,15 +69,36 @@ void tryWrite(char const *buf, size_t size, off_t offset); size_t writeToBuffer(char const *buf, size_t size); - void writeToDisk(const SlotId nextSlot); - void writeBufToDisk(const SlotId nextSlot, const bool eof, const bool lastWrite); + void writeToDisk(); + void callReaderBack(const char *buf, int rlen); void callBack(int errflag); Rock::SwapDir::Pointer dir; ///< swap dir that initiated I/O const size_t slotSize; ///< db cell size int64_t objOffset; ///< object offset for current db slot - SlotId sidCurrent; ///< ID of the db slot currently being read or written + + /// The very first entry slot. Usually the same as anchor.first, + /// but writers set anchor.first only after the first write is done. + SlotId sidFirst; + + /// Unused by readers. + /// For writers, the slot pointing (via .next) to sidCurrent. + SlotId sidPrevious; + + /// For readers, the db slot currently being read from disk. + /// For writers, the reserved db slot currently being filled (to be written). + SlotId sidCurrent; + + /// Unused by readers. + /// For writers, the reserved db slot that sidCurrent.next will point to. + SlotId sidNext; + + /// the number of read or write requests we sent to theFile + uint64_t requestsSent; + + /// the number of successful responses we received from theFile + uint64_t repliesReceived; RefCount theFile; // "file" responsible for this I/O MemBlob theBuf; // use for write content accumulation only diff -u -r -N squid-4.8/src/fs/rock/RockSwapDir.cc squid-4.9/src/fs/rock/RockSwapDir.cc --- squid-4.8/src/fs/rock/RockSwapDir.cc 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/src/fs/rock/RockSwapDir.cc 2019-11-06 08:14:40.000000000 +1300 @@ -834,16 +834,15 @@ ReadRequest *request = dynamic_cast(r.getRaw()); assert(request); IoState::Pointer sio = request->sio; - - if (errflag == DISK_OK && rlen > 0) - sio->offset_ += rlen; - - sio->callReaderBack(r->buf, rlen); + sio->handleReadCompletion(*request, rlen, errflag); } void Rock::SwapDir::writeCompleted(int errflag, size_t, RefCount< ::WriteRequest> r) { + // TODO: Move details into IoState::handleWriteCompletion() after figuring + // out how to deal with map access. See readCompleted(). + Rock::WriteRequest *request = dynamic_cast(r.getRaw()); assert(request); assert(request->sio != NULL); @@ -852,7 +851,7 @@ // quit if somebody called IoState::close() while we were waiting if (!sio.stillWaiting()) { debugs(79, 3, "ignoring closed entry " << sio.swap_filen); - noteFreeMapSlice(request->sidNext); + noteFreeMapSlice(request->sidCurrent); return; } @@ -860,7 +859,7 @@ if (errflag != DISK_OK) handleWriteCompletionProblem(errflag, *request); - else if (droppedEarlierRequest(*request)) + else if (!sio.expectedReply(request->id)) handleWriteCompletionProblem(DISK_ERROR, *request); else handleWriteCompletionSuccess(*request); @@ -877,15 +876,24 @@ sio.splicingPoint = request.sidCurrent; // do not increment sio.offset_ because we do it in sio->write() - // finalize the shared slice info after writing slice contents to disk + assert(sio.writeableAnchor_); + if (sio.writeableAnchor_->start < 0) { // wrote the first slot + Must(request.sidPrevious < 0); + sio.writeableAnchor_->start = request.sidCurrent; + } else { + Must(request.sidPrevious >= 0); + map->writeableSlice(sio.swap_filen, request.sidPrevious).next = request.sidCurrent; + } + + // finalize the shared slice info after writing slice contents to disk; + // the chain gets possession of the slice we were writing Ipc::StoreMap::Slice &slice = map->writeableSlice(sio.swap_filen, request.sidCurrent); slice.size = request.len - sizeof(DbCellHeader); - slice.next = request.sidNext; + Must(slice.next < 0); if (request.eof) { assert(sio.e); - assert(sio.writeableAnchor_); if (sio.touchingStoreEntry()) { sio.e->swap_file_sz = sio.writeableAnchor_->basics.swap_file_sz = sio.offset_; @@ -904,7 +912,7 @@ { auto &sio = *request.sio; - noteFreeMapSlice(request.sidNext); + noteFreeMapSlice(request.sidCurrent); writeError(sio); sio.finishedWriting(errflag); @@ -925,23 +933,6 @@ // All callers must also call IoState callback, to propagate the error. } -/// whether the disk has dropped at least one of the previous write requests -bool -Rock::SwapDir::droppedEarlierRequest(const WriteRequest &request) const -{ - const auto &sio = *request.sio; - assert(sio.writeableAnchor_); - const Ipc::StoreMapSliceId expectedSliceId = sio.splicingPoint < 0 ? - sio.writeableAnchor_->start : - map->writeableSlice(sio.swap_filen, sio.splicingPoint).next; - if (expectedSliceId != request.sidCurrent) { - debugs(79, 3, "yes; expected " << expectedSliceId << ", but got " << request.sidCurrent); - return true; - } - - return false; -} - void Rock::SwapDir::updateHeaders(StoreEntry *updatedE) { diff -u -r -N squid-4.8/src/fs/rock/RockSwapDir.h squid-4.9/src/fs/rock/RockSwapDir.h --- squid-4.8/src/fs/rock/RockSwapDir.h 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/src/fs/rock/RockSwapDir.h 2019-11-06 08:14:40.000000000 +1300 @@ -140,7 +140,6 @@ void createError(const char *const msg); void handleWriteCompletionSuccess(const WriteRequest &request); void handleWriteCompletionProblem(const int errflag, const WriteRequest &request); - bool droppedEarlierRequest(const WriteRequest &request) const; DiskIOStrategy *io; RefCount theFile; ///< cache storage for this cache_dir diff -u -r -N squid-4.8/src/htcp.cc squid-4.9/src/htcp.cc --- squid-4.8/src/htcp.cc 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/src/htcp.cc 2019-11-06 08:14:40.000000000 +1300 @@ -674,7 +674,7 @@ method.HttpRequestMethodXXX(s->method); const MasterXaction::Pointer mx = new MasterXaction(XactionInitiator::initHtcp); - s->request = HttpRequest::FromUrl(s->uri, mx, method == Http::METHOD_NONE ? HttpRequestMethod(Http::METHOD_GET) : method); + s->request = HttpRequest::FromUrlXXX(s->uri, mx, method == Http::METHOD_NONE ? HttpRequestMethod(Http::METHOD_GET) : method); if (!s->request) { debugs(31, 3, "failed to create request. Invalid URI?"); return nil; diff -u -r -N squid-4.8/src/http/one/Parser.cc squid-4.9/src/http/one/Parser.cc --- squid-4.8/src/http/one/Parser.cc 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/src/http/one/Parser.cc 2019-11-06 08:14:40.000000000 +1300 @@ -210,18 +210,19 @@ return true; } -// arbitrary maximum-length for headers which can be found by Http1Parser::getHeaderField() +// arbitrary maximum-length for headers which can be found by Http1Parser::getHostHeaderField() #define GET_HDR_SZ 1024 // BUG: returns only the first header line with given name, // ignores multi-line headers and obs-fold headers char * -Http::One::Parser::getHeaderField(const char *name) +Http::One::Parser::getHostHeaderField() { - if (!headerBlockSize() || !name) + if (!headerBlockSize()) return NULL; LOCAL_ARRAY(char, header, GET_HDR_SZ); + const char *name = "Host"; const int namelen = strlen(name); debugs(25, 5, "looking for " << name); @@ -256,6 +257,11 @@ // prevent buffer overrun on char header[]; p.chop(0, sizeof(header)-1); + // currently only used for pre-parse Host header, ensure valid domain[:port] or ip[:port] + static const auto hostChars = CharacterSet("host",":[].-_") + CharacterSet::ALPHA + CharacterSet::DIGIT; + if (p.findFirstNotOf(hostChars) != SBuf::npos) + break; // error. line contains character not accepted in Host header + // return the header field-value SBufToCstring(header, p); debugs(25, 5, "returning " << header); diff -u -r -N squid-4.8/src/http/one/Parser.h squid-4.9/src/http/one/Parser.h --- squid-4.8/src/http/one/Parser.h 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/src/http/one/Parser.h 2019-11-06 08:14:40.000000000 +1300 @@ -78,7 +78,7 @@ const AnyP::ProtocolVersion & messageProtocol() const {return msgProtocol_;} /** - * Scan the mime header block (badly) for a header with the given name. + * Scan the mime header block (badly) for a Host header. * * BUG: omits lines when searching for headers with obs-fold or multiple entries. * @@ -86,7 +86,7 @@ * * \return A pointer to a field-value of the first matching field-name, or NULL. */ - char *getHeaderField(const char *name); + char *getHostHeaderField(); /// the remaining unprocessed section of buffer const SBuf &remaining() const {return buf_;} diff -u -r -N squid-4.8/src/http/url_rewriters/LFS/url_lfs_rewrite.8 squid-4.9/src/http/url_rewriters/LFS/url_lfs_rewrite.8 --- squid-4.8/src/http/url_rewriters/LFS/url_lfs_rewrite.8 2019-07-10 07:25:14.000000000 +1200 +++ squid-4.9/src/http/url_rewriters/LFS/url_lfs_rewrite.8 2019-11-06 08:27:25.000000000 +1300 @@ -133,7 +133,7 @@ .\" ======================================================================== .\" .IX Title "URL_LFS_REWRITE 8" -.TH URL_LFS_REWRITE 8 "2019-07-09" "perl v5.28.1" "User Contributed Perl Documentation" +.TH URL_LFS_REWRITE 8 "2019-11-05" "perl v5.28.1" "User Contributed Perl Documentation" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l diff -u -r -N squid-4.8/src/HttpHeader.cc squid-4.9/src/HttpHeader.cc --- squid-4.8/src/HttpHeader.cc 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/src/HttpHeader.cc 2019-11-06 08:14:40.000000000 +1300 @@ -421,15 +421,12 @@ break; /* terminating blank line */ } - HttpHeaderEntry *e; - if ((e = HttpHeaderEntry::parse(field_start, field_end)) == NULL) { + const auto e = HttpHeaderEntry::parse(field_start, field_end, owner); + if (!e) { debugs(55, warnOnError, "WARNING: unparseable HTTP header field {" << getStringPrefix(field_start, field_end-field_start) << "}"); debugs(55, warnOnError, " in {" << getStringPrefix(header_start, hdrLen) << "}"); - if (Config.onoff.relaxed_header_parser) - continue; - PROF_stop(HttpHeaderParse); clean(); return 0; @@ -1386,7 +1383,7 @@ /* parses and inits header entry, returns true/false */ HttpHeaderEntry * -HttpHeaderEntry::parse(const char *field_start, const char *field_end) +HttpHeaderEntry::parse(const char *field_start, const char *field_end, const http_hdr_owner_type msgType) { /* note: name_start == field_start */ const char *name_end = (const char *)memchr(field_start, ':', field_end - field_start); @@ -1403,19 +1400,41 @@ if (name_len > 65534) { /* String must be LESS THAN 64K and it adds a terminating NULL */ - debugs(55, DBG_IMPORTANT, "WARNING: ignoring header name of " << name_len << " bytes"); + // TODO: update this to show proper name_len in Raw markup, but not print all that + debugs(55, 2, "ignoring huge header field (" << Raw("field_start", field_start, 100) << "...)"); return NULL; } - if (Config.onoff.relaxed_header_parser && xisspace(field_start[name_len - 1])) { + /* + * RFC 7230 section 3.2.4: + * "No whitespace is allowed between the header field-name and colon. + * ... + * A server MUST reject any received request message that contains + * whitespace between a header field-name and colon with a response code + * of 400 (Bad Request). A proxy MUST remove any such whitespace from a + * response message before forwarding the message downstream." + */ + if (xisspace(field_start[name_len - 1])) { + + if (msgType == hoRequest) + return nullptr; + + // for now, also let relaxed parser remove this BWS from any non-HTTP messages + const bool stripWhitespace = (msgType == hoReply) || + Config.onoff.relaxed_header_parser; + if (!stripWhitespace) + return nullptr; // reject if we cannot strip + debugs(55, Config.onoff.relaxed_header_parser <= 0 ? 1 : 2, "NOTICE: Whitespace after header name in '" << getStringPrefix(field_start, field_end-field_start) << "'"); while (name_len > 0 && xisspace(field_start[name_len - 1])) --name_len; - if (!name_len) + if (!name_len) { + debugs(55, 2, "found header with only whitespace for name"); return NULL; + } } /* now we know we can parse it */ @@ -1448,11 +1467,7 @@ if (field_end - value_start > 65534) { /* String must be LESS THAN 64K and it adds a terminating NULL */ - debugs(55, DBG_IMPORTANT, "WARNING: ignoring '" << name << "' header of " << (field_end - value_start) << " bytes"); - - if (id == Http::HdrType::OTHER) - name.clean(); - + debugs(55, 2, "WARNING: found '" << name << "' header of " << (field_end - value_start) << " bytes"); return NULL; } diff -u -r -N squid-4.8/src/HttpHeader.h squid-4.9/src/HttpHeader.h --- squid-4.8/src/HttpHeader.h 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/src/HttpHeader.h 2019-11-06 08:14:40.000000000 +1300 @@ -54,7 +54,7 @@ public: HttpHeaderEntry(Http::HdrType id, const char *name, const char *value); ~HttpHeaderEntry(); - static HttpHeaderEntry *parse(const char *field_start, const char *field_end); + static HttpHeaderEntry *parse(const char *field_start, const char *field_end, const http_hdr_owner_type msgType); HttpHeaderEntry *clone() const; void packInto(Packable *p) const; int getInt() const; diff -u -r -N squid-4.8/src/HttpRequest.cc squid-4.9/src/HttpRequest.cc --- squid-4.8/src/HttpRequest.cc 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/src/HttpRequest.cc 2019-11-06 08:14:40.000000000 +1300 @@ -327,15 +327,7 @@ if (end < start) // missing URI return false; - char save = *end; - - * (char *) end = '\0'; // temp terminate URI, XXX dangerous? - - const bool ret = url.parse(method, start); - - * (char *) end = save; - - return ret; + return url.parse(method, SBuf(start, size_t(end-start))); } /* swaps out request using httpRequestPack */ @@ -519,7 +511,7 @@ * If the request cannot be created cleanly, NULL is returned */ HttpRequest * -HttpRequest::FromUrl(const char * url, const MasterXaction::Pointer &mx, const HttpRequestMethod& method) +HttpRequest::FromUrl(const SBuf &url, const MasterXaction::Pointer &mx, const HttpRequestMethod& method) { std::unique_ptr req(new HttpRequest(mx)); if (req->url.parse(method, url)) { @@ -529,6 +521,12 @@ return nullptr; } +HttpRequest * +HttpRequest::FromUrlXXX(const char * url, const MasterXaction::Pointer &mx, const HttpRequestMethod& method) +{ + return FromUrl(SBuf(url), mx, method); +} + /** * Are responses to this request possible cacheable ? * If false then no matter what the response must not be cached. diff -u -r -N squid-4.8/src/HttpRequest.h squid-4.9/src/HttpRequest.h --- squid-4.8/src/HttpRequest.h 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/src/HttpRequest.h 2019-11-06 08:14:40.000000000 +1300 @@ -205,7 +205,10 @@ static void httpRequestPack(void *obj, Packable *p); - static HttpRequest * FromUrl(const char * url, const MasterXaction::Pointer &, const HttpRequestMethod &method = Http::METHOD_GET); + static HttpRequest * FromUrl(const SBuf &url, const MasterXaction::Pointer &, const HttpRequestMethod &method = Http::METHOD_GET); + + /// \deprecated use SBuf variant instead + static HttpRequest * FromUrlXXX(const char * url, const MasterXaction::Pointer &, const HttpRequestMethod &method = Http::METHOD_GET); ConnStateData *pinnedConnection(); diff -u -r -N squid-4.8/src/icmp/net_db.cc squid-4.9/src/icmp/net_db.cc --- squid-4.8/src/icmp/net_db.cc 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/src/icmp/net_db.cc 2019-11-06 08:14:40.000000000 +1300 @@ -1285,7 +1285,7 @@ char *uri = internalRemoteUri(p->secure.encryptTransport, p->host, p->http_port, "/squid-internal-dynamic/", netDB); debugs(38, 3, "Requesting '" << uri << "'"); const MasterXaction::Pointer mx = new MasterXaction(XactionInitiator::initIcmp); - HttpRequest *req = HttpRequest::FromUrl(uri, mx); + auto req = HttpRequest::FromUrlXXX(uri, mx); if (!req) { debugs(38, DBG_IMPORTANT, MYNAME << ": Bad URI " << uri); diff -u -r -N squid-4.8/src/icp_v2.cc squid-4.9/src/icp_v2.cc --- squid-4.8/src/icp_v2.cc 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/src/icp_v2.cc 2019-11-06 08:14:40.000000000 +1300 @@ -440,9 +440,9 @@ return NULL; } - HttpRequest *result; const MasterXaction::Pointer mx = new MasterXaction(XactionInitiator::initIcp); - if ((result = HttpRequest::FromUrl(url, mx)) == NULL) + auto *result = HttpRequest::FromUrlXXX(url, mx); + if (!result) icpCreateAndSend(ICP_ERR, 0, url, reqnum, 0, fd, from); return result; diff -u -r -N squid-4.8/src/internal.cc squid-4.9/src/internal.cc --- squid-4.8/src/internal.cc 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/src/internal.cc 2019-11-06 08:14:40.000000000 +1300 @@ -98,13 +98,9 @@ /* * append the domain in order to mirror the requests with appended - * domains + * domains. If that fails, just use the hostname anyway. */ - - /* For IPv6 addresses also check for a colon */ - if (Config.appendDomain && !strchr(lc_host, '.') && !strchr(lc_host, ':')) - strncat(lc_host, Config.appendDomain, SQUIDHOSTNAMELEN - - strlen(lc_host) - 1); + (void)urlAppendDomain(lc_host); /* build URI */ AnyP::Uri tmp(AnyP::PROTO_HTTP); diff -u -r -N squid-4.8/src/log/DB/log_db_daemon.8 squid-4.9/src/log/DB/log_db_daemon.8 --- squid-4.8/src/log/DB/log_db_daemon.8 2019-07-10 07:25:14.000000000 +1200 +++ squid-4.9/src/log/DB/log_db_daemon.8 2019-11-06 08:27:25.000000000 +1300 @@ -133,7 +133,7 @@ .\" ======================================================================== .\" .IX Title "LOG_DB_DAEMON 8" -.TH LOG_DB_DAEMON 8 "2019-07-09" "perl v5.28.1" "User Contributed Perl Documentation" +.TH LOG_DB_DAEMON 8 "2019-11-05" "perl v5.28.1" "User Contributed Perl Documentation" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l diff -u -r -N squid-4.8/src/Makefile.am squid-4.9/src/Makefile.am --- squid-4.8/src/Makefile.am 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/src/Makefile.am 2019-11-06 08:14:40.000000000 +1300 @@ -1136,6 +1136,7 @@ acl/libstate.la \ acl/libapi.la \ anyp/libanyp.la \ + parser/libparser.la \ base/libbase.la \ ip/libip.la \ ipc/libipc.la \ diff -u -r -N squid-4.8/src/Makefile.in squid-4.9/src/Makefile.in --- squid-4.8/src/Makefile.in 2019-07-10 07:16:47.000000000 +1200 +++ squid-4.9/src/Makefile.in 2019-11-06 08:19:02.000000000 +1300 @@ -489,9 +489,9 @@ tests_testACLMaxUserIP_DEPENDENCIES = libsquid.la helper/libhelper.la \ http/libhttp.la parser/libparser.la $(AUTH_ACL_LIBS) \ ident/libident.la acl/libacls.la eui/libeui.la acl/libstate.la \ - acl/libapi.la anyp/libanyp.la base/libbase.la ip/libip.la \ - ipc/libipc.la mgr/libmgr.la sbuf/libsbuf.la \ - $(top_builddir)/lib/libmisccontainers.la \ + acl/libapi.la anyp/libanyp.la parser/libparser.la \ + base/libbase.la ip/libip.la ipc/libipc.la mgr/libmgr.la \ + sbuf/libsbuf.la $(top_builddir)/lib/libmisccontainers.la \ $(top_builddir)/lib/libmiscencoding.la \ $(top_builddir)/lib/libmiscutil.la $(am__DEPENDENCIES_1) \ $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_1) \ @@ -3585,6 +3585,7 @@ acl/libstate.la \ acl/libapi.la \ anyp/libanyp.la \ + parser/libparser.la \ base/libbase.la \ ip/libip.la \ ipc/libipc.la \ diff -u -r -N squid-4.8/src/MemStore.cc squid-4.9/src/MemStore.cc --- squid-4.8/src/MemStore.cc 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/src/MemStore.cc 2019-11-06 08:14:40.000000000 +1300 @@ -479,7 +479,7 @@ wasEof = anchor.complete() && slice.next < 0; const Ipc::StoreMapSlice::Size wasSize = slice.size; - debugs(20, 9, "entry " << index << " slice " << sid << " eof " << + debugs(20, 8, "entry " << index << " slice " << sid << " eof " << wasEof << " wasSize " << wasSize << " <= " << anchor.basics.swap_file_sz << " sliceOffset " << sliceOffset << " mem.endOffset " << e.mem_obj->endOffset()); @@ -497,7 +497,7 @@ page + prefixSize); if (!copyFromShmSlice(e, sliceBuf, wasEof)) return false; - debugs(20, 9, "entry " << index << " copied slice " << sid << + debugs(20, 8, "entry " << index << " copied slice " << sid << " from " << extra.page << '+' << prefixSize); } // else skip a [possibly incomplete] slice that we copied earlier @@ -521,7 +521,7 @@ return true; } - debugs(20, 7, "mem-loaded all " << e.mem_obj->object_sz << '/' << + debugs(20, 5, "mem-loaded all " << e.mem_obj->endOffset() << '/' << anchor.basics.swap_file_sz << " bytes of " << e); // from StoreEntry::complete() diff -u -r -N squid-4.8/src/mgr/Inquirer.cc squid-4.9/src/mgr/Inquirer.cc --- squid-4.8/src/mgr/Inquirer.cc 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/src/mgr/Inquirer.cc 2019-11-06 08:14:40.000000000 +1300 @@ -76,7 +76,7 @@ if (strands.empty()) { const char *url = aggrAction->command().params.httpUri.termedBuf(); const MasterXaction::Pointer mx = new MasterXaction(XactionInitiator::initIpc); - HttpRequest *req = HttpRequest::FromUrl(url, mx); + auto *req = HttpRequest::FromUrlXXX(url, mx); ErrorState err(ERR_INVALID_URL, Http::scNotFound, req); std::unique_ptr reply(err.BuildHttpReply()); replyBuf.reset(reply->pack()); diff -u -r -N squid-4.8/src/mime.cc squid-4.9/src/mime.cc --- squid-4.8/src/mime.cc 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/src/mime.cc 2019-11-06 08:14:40.000000000 +1300 @@ -402,7 +402,7 @@ /* fill `e` with a canned 2xx response object */ const MasterXaction::Pointer mx = new MasterXaction(XactionInitiator::initIcon); - HttpRequest *r = HttpRequest::FromUrl(url_, mx); + auto r = HttpRequest::FromUrlXXX(url_, mx); if (!r) fatalf("mimeLoadIcon: cannot parse internal URL: %s", url_); diff -u -r -N squid-4.8/src/neighbors.cc squid-4.9/src/neighbors.cc --- squid-4.8/src/neighbors.cc 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/src/neighbors.cc 2019-11-06 08:14:40.000000000 +1300 @@ -1373,7 +1373,7 @@ p->in_addr.toUrl(url+7, MAX_URL -8 ); strcat(url, "/"); const MasterXaction::Pointer mx = new MasterXaction(XactionInitiator::initPeerMcast); - HttpRequest *req = HttpRequest::FromUrl(url, mx); + auto *req = HttpRequest::FromUrlXXX(url, mx); assert(req != nullptr); StoreEntry *fake = storeCreateEntry(url, url, RequestFlags(), Http::METHOD_GET); psstate = new ps_state; diff -u -r -N squid-4.8/src/peer_digest.cc squid-4.9/src/peer_digest.cc --- squid-4.8/src/peer_digest.cc 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/src/peer_digest.cc 2019-11-06 08:14:40.000000000 +1300 @@ -327,7 +327,7 @@ debugs(72, 2, url); const MasterXaction::Pointer mx = new MasterXaction(XactionInitiator::initCacheDigest); - req = HttpRequest::FromUrl(url, mx); + req = HttpRequest::FromUrlXXX(url, mx); assert(req); diff -u -r -N squid-4.8/src/security/cert_validators/fake/security_fake_certverify.8 squid-4.9/src/security/cert_validators/fake/security_fake_certverify.8 --- squid-4.8/src/security/cert_validators/fake/security_fake_certverify.8 2019-07-10 07:25:14.000000000 +1200 +++ squid-4.9/src/security/cert_validators/fake/security_fake_certverify.8 2019-11-06 08:27:26.000000000 +1300 @@ -133,7 +133,7 @@ .\" ======================================================================== .\" .IX Title "SECURITY_FAKE_CERTVERIFY 8" -.TH SECURITY_FAKE_CERTVERIFY 8 "2019-07-09" "perl v5.28.1" "User Contributed Perl Documentation" +.TH SECURITY_FAKE_CERTVERIFY 8 "2019-11-05" "perl v5.28.1" "User Contributed Perl Documentation" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l diff -u -r -N squid-4.8/src/security/Handshake.cc squid-4.9/src/security/Handshake.cc --- squid-4.8/src/security/Handshake.cc 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/src/security/Handshake.cc 2019-11-06 08:14:40.000000000 +1300 @@ -244,22 +244,23 @@ Must(record.fragment.length() || record.type == ContentType::ctApplicationData); if (currentContentType != record.type) { + parseMessages(); Must(tkMessages.atEnd()); // no currentContentType leftovers fragments = record.fragment; - tkMessages.reset(fragments, true); // true because more fragments may come currentContentType = record.type; } else { fragments.append(record.fragment); - tkMessages.reinput(fragments, true); // true because more fragments may come - tkMessages.rollback(); } - parseMessages(); + + if (tkRecords.atEnd() && !done) + parseMessages(); } /// parses one or more "higher-level protocol" frames of currentContentType void Security::HandshakeParser::parseMessages() { + tkMessages.reset(fragments, false); for (; !tkMessages.atEnd(); tkMessages.commit()) { switch (currentContentType) { case ContentType::ctChangeCipherSpec: diff -u -r -N squid-4.8/src/security/KeyData.cc squid-4.9/src/security/KeyData.cc --- squid-4.8/src/security/KeyData.cc 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/src/security/KeyData.cc 2019-11-06 08:14:40.000000000 +1300 @@ -119,14 +119,15 @@ } #endif // checks that the chained certs are actually part of a chain for validating cert - if (X509_check_issued(ca, latestCert.get()) == X509_V_OK) { + const auto checkCode = X509_check_issued(ca, latestCert.get()); + if (checkCode == X509_V_OK) { debugs(83, DBG_PARSE_NOTE(3), "Adding issuer CA: " << nameStr); // OpenSSL API requires that we order certificates such that the // chain can be appended directly into the on-wire traffic. latestCert = CertPointer(ca); chain.emplace_front(latestCert); } else { - debugs(83, DBG_PARSE_NOTE(2), "Ignoring non-issuer CA from " << certFile << ": " << nameStr); + debugs(83, DBG_PARSE_NOTE(2), certFile << ": Ignoring non-issuer CA " << nameStr << ": " << X509_verify_cert_error_string(checkCode) << " (" << checkCode << ")"); } OPENSSL_free(nameStr); } diff -u -r -N squid-4.8/src/security/PeerOptions.h squid-4.9/src/security/PeerOptions.h --- squid-4.8/src/security/PeerOptions.h 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/src/security/PeerOptions.h 2019-11-06 08:14:40.000000000 +1300 @@ -115,7 +115,7 @@ #elif USE_GNUTLS debugs(83, 5, "gnutls_certificate_credentials construct, this=" << (void*)ctx); return Security::ContextPointer(ctx, [](gnutls_certificate_credentials_t p) { - debugs(83, 0, "gnutls_certificate_credentials destruct this=" << (void*)p); + debugs(83, 5, "gnutls_certificate_credentials destruct, this=" << (void*)p); gnutls_certificate_free_credentials(p); }); #else diff -u -r -N squid-4.8/src/servers/FtpServer.cc squid-4.9/src/servers/FtpServer.cc --- squid-4.8/src/servers/FtpServer.cc 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/src/servers/FtpServer.cc 2019-11-06 08:14:40.000000000 +1300 @@ -726,7 +726,7 @@ calcUri(path); MasterXaction::Pointer mx = new MasterXaction(XactionInitiator::initClient); mx->tcpClient = clientConnection; - HttpRequest *const request = HttpRequest::FromUrl(uri.c_str(), mx, method); + auto * const request = HttpRequest::FromUrl(uri, mx, method); if (!request) { debugs(33, 5, "Invalid FTP URL: " << uri); uri.clear(); diff -u -r -N squid-4.8/src/servers/Http1Server.cc squid-4.9/src/servers/Http1Server.cc --- squid-4.8/src/servers/Http1Server.cc 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/src/servers/Http1Server.cc 2019-11-06 08:14:40.000000000 +1300 @@ -74,14 +74,18 @@ { PROF_start(HttpServer_parseOneRequest); + // reset because the protocol may have changed if this is the first request + // and because we never bypass parsing failures of N+1st same-proto request + preservingClientData_ = shouldPreserveClientData(); + // parser is incremental. Generate new parser state if we, // a) do not have one already // b) have completed the previous request parsing already if (!parser_ || !parser_->needsMoreData()) - parser_ = new Http1::RequestParser(mayTunnelUnsupportedProto()); + parser_ = new Http1::RequestParser(preservingClientData_); /* Process request */ - Http::Stream *context = parseHttpRequest(this, parser_); + Http::Stream *context = parseHttpRequest(parser_); PROF_stop(HttpServer_parseOneRequest); return context; @@ -135,7 +139,8 @@ // TODO: move URL parse into Http Parser and INVALID_URL into the above parse error handling MasterXaction::Pointer mx = new MasterXaction(XactionInitiator::initClient); mx->tcpClient = clientConnection; - if ((request = HttpRequest::FromUrl(http->uri, mx, parser_->method())) == NULL) { + request = HttpRequest::FromUrlXXX(http->uri, mx, parser_->method()); + if (!request) { debugs(33, 5, "Invalid URL: " << http->uri); // setReplyToError() requires log_uri http->setLogUriToRawUri(http->uri, parser_->method()); diff -u -r -N squid-4.8/src/ssl/cert_validate_message.cc squid-4.9/src/ssl/cert_validate_message.cc --- squid-4.8/src/ssl/cert_validate_message.cc 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/src/ssl/cert_validate_message.cc 2019-11-06 08:14:40.000000000 +1300 @@ -171,7 +171,7 @@ return false; } - param = value + value_len +1; + param = value + value_len; } /*Run through parsed errors to check for errors*/ diff -u -r -N squid-4.8/src/ssl/gadgets.cc squid-4.9/src/ssl/gadgets.cc --- squid-4.8/src/ssl/gadgets.cc 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/src/ssl/gadgets.cc 2019-11-06 08:14:40.000000000 +1300 @@ -504,7 +504,7 @@ if (aTime) { if (!X509_set1_notAfter(cert.get(), aTime)) return false; - } else if (!X509_gmtime_adj(X509_getm_notAfter(cert.get()), 60*60*24*356*3)) + } else if (!X509_gmtime_adj(X509_getm_notAfter(cert.get()), 60*60*24*365*3)) return false; int addedExtensions = 0; diff -u -r -N squid-4.8/src/store/id_rewriters/file/storeid_file_rewrite.8 squid-4.9/src/store/id_rewriters/file/storeid_file_rewrite.8 --- squid-4.8/src/store/id_rewriters/file/storeid_file_rewrite.8 2019-07-10 07:25:12.000000000 +1200 +++ squid-4.9/src/store/id_rewriters/file/storeid_file_rewrite.8 2019-11-06 08:27:24.000000000 +1300 @@ -133,7 +133,7 @@ .\" ======================================================================== .\" .IX Title "STOREID_FILE_REWRITE 8" -.TH STOREID_FILE_REWRITE 8 "2019-07-09" "perl v5.28.1" "User Contributed Perl Documentation" +.TH STOREID_FILE_REWRITE 8 "2019-11-05" "perl v5.28.1" "User Contributed Perl Documentation" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l diff -u -r -N squid-4.8/src/store_client.cc squid-4.9/src/store_client.cc --- squid-4.8/src/store_client.cc 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/src/store_client.cc 2019-11-06 08:14:40.000000000 +1300 @@ -458,6 +458,9 @@ assert(_callback.pending()); debugs(90, 3, "storeClientReadBody: len " << len << ""); + if (len < 0) + return fail(); + if (copyInto.offset == 0 && len > 0 && entry->getReply()->sline.status() == Http::scNone) { /* Our structure ! */ HttpReply *rep = (HttpReply *) entry->getReply(); // bypass const @@ -519,13 +522,8 @@ bool store_client::unpackHeader(char const *buf, ssize_t len) { - int xerrno = errno; // FIXME: where does errno come from? debugs(90, 3, "store_client::unpackHeader: len " << len << ""); - - if (len < 0) { - debugs(90, 3, "WARNING: unpack error: " << xstrerr(xerrno)); - return false; - } + assert(len >= 0); int swap_hdr_sz = 0; tlv *tlv_list = nullptr; @@ -575,6 +573,9 @@ if (!object_ok) return; + if (len < 0) + return fail(); + if (!unpackHeader(buf, len)) { fail(); return; diff -u -r -N squid-4.8/src/store_digest.cc squid-4.9/src/store_digest.cc --- squid-4.8/src/store_digest.cc 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/src/store_digest.cc 2019-11-06 08:14:40.000000000 +1300 @@ -414,7 +414,7 @@ const char *url = internalLocalUri("/squid-internal-periodic/", SBuf(StoreDigestFileName)); const MasterXaction::Pointer mx = new MasterXaction(XactionInitiator::initCacheDigest); - auto req = HttpRequest::FromUrl(url, mx); + auto req = HttpRequest::FromUrlXXX(url, mx); RequestFlags flags; flags.cachable = true; diff -u -r -N squid-4.8/src/tests/stub_HttpRequest.cc squid-4.9/src/tests/stub_HttpRequest.cc --- squid-4.8/src/tests/stub_HttpRequest.cc 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/src/tests/stub_HttpRequest.cc 2019-11-06 08:14:40.000000000 +1300 @@ -47,7 +47,8 @@ void HttpRequest::swapOut(StoreEntry *) STUB void HttpRequest::pack(Packable *) const STUB void HttpRequest::httpRequestPack(void *, Packable *) STUB -HttpRequest * HttpRequest::FromUrl(const char *, const MasterXaction::Pointer &, const HttpRequestMethod &) STUB_RETVAL(NULL) +HttpRequest * HttpRequest::FromUrl(const SBuf &, const MasterXaction::Pointer &, const HttpRequestMethod &) STUB_RETVAL(nullptr) +HttpRequest * HttpRequest::FromUrlXXX(const char *, const MasterXaction::Pointer &, const HttpRequestMethod &) STUB_RETVAL(nullptr) ConnStateData *HttpRequest::pinnedConnection() STUB_RETVAL(NULL) const SBuf HttpRequest::storeId() STUB_RETVAL(SBuf(".")) void HttpRequest::ignoreRange(const char *) STUB diff -u -r -N squid-4.8/src/tests/stub_libanyp.cc squid-4.9/src/tests/stub_libanyp.cc --- squid-4.8/src/tests/stub_libanyp.cc 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/src/tests/stub_libanyp.cc 2019-11-06 08:14:40.000000000 +1300 @@ -14,7 +14,7 @@ #include "anyp/Uri.h" AnyP::Uri::Uri(AnyP::UriScheme const &) {STUB} void AnyP::Uri::touch() STUB -bool AnyP::Uri::parse(const HttpRequestMethod&, const char *) STUB_RETVAL(true) +bool AnyP::Uri::parse(const HttpRequestMethod&, const SBuf &) STUB_RETVAL(true) void AnyP::Uri::host(const char *) STUB static SBuf nil; const SBuf &AnyP::Uri::path() const STUB_RETVAL(nil) diff -u -r -N squid-4.8/src/tests/testHttpRequest.cc squid-4.9/src/tests/testHttpRequest.cc --- squid-4.8/src/tests/testHttpRequest.cc 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/src/tests/testHttpRequest.cc 2019-11-06 08:14:40.000000000 +1300 @@ -45,60 +45,55 @@ { /* vanilla url, implict method */ unsigned short expected_port; - char * url = xstrdup("http://foo:90/bar"); + SBuf url("http://foo:90/bar"); const MasterXaction::Pointer mx = new MasterXaction(XactionInitiator::initClient); HttpRequest *aRequest = HttpRequest::FromUrl(url, mx); expected_port = 90; + CPPUNIT_ASSERT(aRequest != nullptr); CPPUNIT_ASSERT_EQUAL(expected_port, aRequest->url.port()); CPPUNIT_ASSERT(aRequest->method == Http::METHOD_GET); CPPUNIT_ASSERT_EQUAL(String("foo"), String(aRequest->url.host())); CPPUNIT_ASSERT_EQUAL(SBuf("/bar"), aRequest->url.path()); CPPUNIT_ASSERT_EQUAL(AnyP::PROTO_HTTP, static_cast(aRequest->url.getScheme())); - CPPUNIT_ASSERT_EQUAL(String("http://foo:90/bar"), String(url)); - xfree(url); /* vanilla url */ - url = xstrdup("http://foo:90/bar"); + url = "http://foo:90/bar"; aRequest = HttpRequest::FromUrl(url, mx, Http::METHOD_GET); expected_port = 90; + CPPUNIT_ASSERT(aRequest != nullptr); CPPUNIT_ASSERT_EQUAL(expected_port, aRequest->url.port()); CPPUNIT_ASSERT(aRequest->method == Http::METHOD_GET); CPPUNIT_ASSERT_EQUAL(String("foo"), String(aRequest->url.host())); CPPUNIT_ASSERT_EQUAL(SBuf("/bar"), aRequest->url.path()); CPPUNIT_ASSERT_EQUAL(AnyP::PROTO_HTTP, static_cast(aRequest->url.getScheme())); - CPPUNIT_ASSERT_EQUAL(String("http://foo:90/bar"), String(url)); - xfree(url); /* vanilla url, different method */ - url = xstrdup("http://foo/bar"); + url = "http://foo/bar"; aRequest = HttpRequest::FromUrl(url, mx, Http::METHOD_PUT); expected_port = 80; + CPPUNIT_ASSERT(aRequest != nullptr); CPPUNIT_ASSERT_EQUAL(expected_port, aRequest->url.port()); CPPUNIT_ASSERT(aRequest->method == Http::METHOD_PUT); CPPUNIT_ASSERT_EQUAL(String("foo"), String(aRequest->url.host())); CPPUNIT_ASSERT_EQUAL(SBuf("/bar"), aRequest->url.path()); CPPUNIT_ASSERT_EQUAL(AnyP::PROTO_HTTP, static_cast(aRequest->url.getScheme())); - CPPUNIT_ASSERT_EQUAL(String("http://foo/bar"), String(url)); - xfree(url); /* a connect url with non-CONNECT data */ HttpRequest *nullRequest = nullptr; - url = xstrdup(":foo/bar"); + url = ":foo/bar"; aRequest = HttpRequest::FromUrl(url, mx, Http::METHOD_CONNECT); - xfree(url); CPPUNIT_ASSERT_EQUAL(nullRequest, aRequest); /* a CONNECT url with CONNECT data */ - url = xstrdup("foo:45"); + url = "foo:45"; aRequest = HttpRequest::FromUrl(url, mx, Http::METHOD_CONNECT); expected_port = 45; + CPPUNIT_ASSERT(aRequest != nullptr); CPPUNIT_ASSERT_EQUAL(expected_port, aRequest->url.port()); CPPUNIT_ASSERT(aRequest->method == Http::METHOD_CONNECT); CPPUNIT_ASSERT_EQUAL(String("foo"), String(aRequest->url.host())); CPPUNIT_ASSERT_EQUAL(SBuf(), aRequest->url.path()); CPPUNIT_ASSERT_EQUAL(AnyP::PROTO_NONE, static_cast(aRequest->url.getScheme())); - CPPUNIT_ASSERT_EQUAL(String("foo:45"), String(url)); - xfree(url); // XXX: check METHOD_NONE input handling } @@ -110,11 +105,10 @@ testHttpRequest::testIPv6HostColonBug() { unsigned short expected_port; - char * url = NULL; HttpRequest *aRequest = NULL; /* valid IPv6 address without port */ - url = xstrdup("http://[2000:800::45]/foo"); + SBuf url("http://[2000:800::45]/foo"); const MasterXaction::Pointer mx = new MasterXaction(XactionInitiator::initClient); aRequest = HttpRequest::FromUrl(url, mx, Http::METHOD_GET); expected_port = 80; @@ -123,11 +117,9 @@ CPPUNIT_ASSERT_EQUAL(String("[2000:800::45]"), String(aRequest->url.host())); CPPUNIT_ASSERT_EQUAL(SBuf("/foo"), aRequest->url.path()); CPPUNIT_ASSERT_EQUAL(AnyP::PROTO_HTTP, static_cast(aRequest->url.getScheme())); - CPPUNIT_ASSERT_EQUAL(String("http://[2000:800::45]/foo"), String(url)); - xfree(url); /* valid IPv6 address with port */ - url = xstrdup("http://[2000:800::45]:90/foo"); + url = "http://[2000:800::45]:90/foo"; aRequest = HttpRequest::FromUrl(url, mx, Http::METHOD_GET); expected_port = 90; CPPUNIT_ASSERT_EQUAL(expected_port, aRequest->url.port()); @@ -135,11 +127,9 @@ CPPUNIT_ASSERT_EQUAL(String("[2000:800::45]"), String(aRequest->url.host())); CPPUNIT_ASSERT_EQUAL(SBuf("/foo"), aRequest->url.path()); CPPUNIT_ASSERT_EQUAL(AnyP::PROTO_HTTP, static_cast(aRequest->url.getScheme())); - CPPUNIT_ASSERT_EQUAL(String("http://[2000:800::45]:90/foo"), String(url)); - xfree(url); /* IPv6 address as invalid (bug trigger) */ - url = xstrdup("http://2000:800::45/foo"); + url = "http://2000:800::45/foo"; aRequest = HttpRequest::FromUrl(url, mx, Http::METHOD_GET); expected_port = 80; CPPUNIT_ASSERT_EQUAL(expected_port, aRequest->url.port()); @@ -147,8 +137,6 @@ CPPUNIT_ASSERT_EQUAL(String("[2000:800::45]"), String(aRequest->url.host())); CPPUNIT_ASSERT_EQUAL(SBuf("/foo"), aRequest->url.path()); CPPUNIT_ASSERT_EQUAL(AnyP::PROTO_HTTP, static_cast(aRequest->url.getScheme())); - CPPUNIT_ASSERT_EQUAL(String("http://2000:800::45/foo"), String(url)); - xfree(url); } void diff -u -r -N squid-4.8/src/urn.cc squid-4.9/src/urn.cc --- squid-4.8/src/urn.cc 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/src/urn.cc 2019-11-06 08:14:40.000000000 +1300 @@ -9,6 +9,7 @@ /* DEBUG: section 52 URN Parsing */ #include "squid.h" +#include "base/TextException.h" #include "cbdata.h" #include "errorpage.h" #include "FwdState.h" @@ -34,7 +35,6 @@ public: void created (StoreEntry *newEntry); void start (HttpRequest *, StoreEntry *); - char *getHost(const SBuf &urlpath); void setUriResFromRequest(HttpRequest *); virtual ~UrnState(); @@ -45,11 +45,8 @@ HttpRequest::Pointer request; HttpRequest::Pointer urlres_r; - struct { - bool force_menu; - } flags; - char reqbuf[URN_REQBUF_SZ]; - int reqofs; + char reqbuf[URN_REQBUF_SZ] = { '\0' }; + int reqofs = 0; private: char *urlres; @@ -73,7 +70,18 @@ UrnState::~UrnState() { - safe_free(urlres); + SWALLOW_EXCEPTIONS({ + if (urlres_e) { + if (sc) + storeUnregister(sc, urlres_e, this); + urlres_e->unlock("~UrnState+res"); + } + + if (entry) + entry->unlock("~UrnState+prime"); + + safe_free(urlres); + }); } static url_entry * @@ -122,35 +130,16 @@ return min_u; } -char * -UrnState::getHost(const SBuf &urlpath) -{ - /** FIXME: this appears to be parsing the URL. *very* badly. */ - /* a proper encapsulated URI/URL type needs to clear this up. */ - size_t p; - if ((p = urlpath.find(':')) != SBuf::npos) - return SBufToCstring(urlpath.substr(0, p-1)); - - return SBufToCstring(urlpath); -} - void UrnState::setUriResFromRequest(HttpRequest *r) { - static const SBuf menu(".menu"); - if (r->url.path().startsWith(menu)) { - r->url.path(r->url.path().substr(5)); // strip prefix "menu." - flags.force_menu = true; - } - - SBuf uri = r->url.path(); + const auto &query = r->url.absolute(); + const auto host = r->url.host(); // TODO: use class AnyP::Uri instead of generating a string and re-parsing LOCAL_ARRAY(char, local_urlres, 4096); - char *host = getHost(uri); - snprintf(local_urlres, 4096, "http://%s/uri-res/N2L?urn:" SQUIDSBUFPH, host, SQUIDSBUFPRINT(uri)); - safe_free(host); + snprintf(local_urlres, 4096, "http://%s/uri-res/N2L?" SQUIDSBUFPH, host, SQUIDSBUFPRINT(query)); safe_free(urlres); - urlres_r = HttpRequest::FromUrl(local_urlres, r->masterXaction); + urlres_r = HttpRequest::FromUrlXXX(local_urlres, r->masterXaction); if (!urlres_r) { debugs(52, 3, "Bad uri-res URL " << local_urlres); @@ -228,14 +217,6 @@ return u1->rtt - u2->rtt; } -static void -urnHandleReplyError(UrnState *urnState, StoreEntry *urlres_e) -{ - urlres_e->unlock("urnHandleReplyError+res"); - urnState->entry->unlock("urnHandleReplyError+prime"); - delete urnState; -} - /* TODO: use the clientStream support for this */ static void urnHandleReply(void *data, StoreIOBuffer result) @@ -258,8 +239,8 @@ debugs(52, 3, "urnHandleReply: Called with size=" << result.length << "."); - if (EBIT_TEST(urlres_e->flags, ENTRY_ABORTED) || result.length == 0 || result.flags.error) { - urnHandleReplyError(urnState, urlres_e); + if (EBIT_TEST(urlres_e->flags, ENTRY_ABORTED) || result.flags.error) { + delete urnState; return; } @@ -268,15 +249,15 @@ /* Handle reqofs being bigger than normal */ if (urnState->reqofs >= URN_REQBUF_SZ) { - urnHandleReplyError(urnState, urlres_e); + delete urnState; return; } /* If we haven't received the entire object (urn), copy more */ - if (urlres_e->store_status == STORE_PENDING && - urnState->reqofs < URN_REQBUF_SZ) { + if (urlres_e->store_status == STORE_PENDING) { + Must(result.length > 0); // zero length ought to imply STORE_OK tempBuffer.offset = urnState->reqofs; - tempBuffer.length = URN_REQBUF_SZ; + tempBuffer.length = URN_REQBUF_SZ - urnState->reqofs; tempBuffer.data = urnState->reqbuf + urnState->reqofs; storeClientCopy(urnState->sc, urlres_e, tempBuffer, @@ -290,7 +271,7 @@ if (0 == k) { debugs(52, DBG_IMPORTANT, "urnHandleReply: didn't find end-of-headers for " << e->url() ); - urnHandleReplyError(urnState, urlres_e); + delete urnState; return; } @@ -306,7 +287,7 @@ err->url = xstrdup(e->url()); errorAppendEntry(e, err); delete rep; - urnHandleReplyError(urnState, urlres_e); + delete urnState; return; } @@ -322,7 +303,7 @@ err = new ErrorState(ERR_URN_RESOLVE, Http::scNotFound, urnState->request.getRaw()); err->url = xstrdup(e->url()); errorAppendEntry(e, err); - urnHandleReplyError(urnState, urlres_e); + delete urnState; return; } @@ -366,9 +347,7 @@ rep = new HttpReply; rep->setHeaders(Http::scFound, NULL, "text/html", mb->contentSize(), 0, squid_curtime); - if (urnState->flags.force_menu) { - debugs(51, 3, "urnHandleReply: forcing menu"); - } else if (min_u) { + if (min_u) { rep->header.putStr(Http::HdrType::LOCATION, min_u->url); } @@ -383,10 +362,7 @@ } safe_free(urls); - /* mb was absorbed in httpBodySet call, so we must not clean it */ - storeUnregister(urnState->sc, urlres_e, urnState); - - urnHandleReplyError(urnState, urlres_e); + delete urnState; } static url_entry * diff -u -r -N squid-4.8/tools/cachemgr.cc squid-4.9/tools/cachemgr.cc --- squid-4.8/tools/cachemgr.cc 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/tools/cachemgr.cc 2019-11-06 08:14:40.000000000 +1300 @@ -8,6 +8,7 @@ #include "squid.h" #include "base64.h" +#include "base/CharacterSet.h" #include "getfullhostname.h" #include "html_quote.h" #include "ip/Address.h" @@ -215,6 +216,21 @@ return ""; } +bool +hostname_check(const char *uri) +{ + static CharacterSet hostChars = CharacterSet("host",".:[]_") + + CharacterSet::ALPHA + CharacterSet::DIGIT; + + const auto limit = strlen(uri); + for (size_t i = 0; i < limit; i++) { + if (!hostChars[uri[i]]) { + return false; + } + } + return true; +} + static void print_trailer(void) { @@ -807,9 +823,15 @@ } else if ((S = req->hostname)) (void) 0; else { - snprintf(buf, sizeof(buf), "Unknown host: %s\n", req->hostname); - error_html(buf); - return 1; + if (hostname_check(req->hostname)) { + snprintf(buf, sizeof(buf), "Unknown Host: %s\n", req->hostname); + error_html(buf); + return 1; + } else { + snprintf(buf, sizeof(buf), "%s\n", "Invalid Hostname"); + error_html(buf); + return 1; + } } S.port(req->port); diff -u -r -N squid-4.8/tools/CharacterSet.cc squid-4.9/tools/CharacterSet.cc --- squid-4.8/tools/CharacterSet.cc 1970-01-01 12:00:00.000000000 +1200 +++ squid-4.9/tools/CharacterSet.cc 2019-11-06 08:27:26.000000000 +1300 @@ -0,0 +1,155 @@ +/* + * Copyright (C) 1996-2019 The Squid Software Foundation and contributors + * + * Squid software is distributed under GPLv2+ license and includes + * contributions from numerous individuals and organizations. + * Please see the COPYING and CONTRIBUTORS files for details. + */ + +#include "squid.h" +#include "base/CharacterSet.h" + +#include +#include +#include + +CharacterSet & +CharacterSet::operator +=(const CharacterSet &src) +{ + Storage::const_iterator s = src.chars_.begin(); + const Storage::const_iterator e = src.chars_.end(); + Storage::iterator d = chars_.begin(); + while (s != e) { + if (*s) + *d = 1; + ++s; + ++d; + } + return *this; +} + +CharacterSet & +CharacterSet::operator -=(const CharacterSet &src) +{ + Storage::const_iterator s = src.chars_.begin(); + const Storage::const_iterator e = src.chars_.end(); + Storage::iterator d = chars_.begin(); + while (s != e) { + if (*s) + *d = 0; + ++s; + ++d; + } + return *this; +} + +CharacterSet & +CharacterSet::add(const unsigned char c) +{ + chars_[static_cast(c)] = 1; + return *this; +} + +CharacterSet & +CharacterSet::addRange(unsigned char low, unsigned char high) +{ + //manual loop splitting is needed to cover case where high is 255 + // otherwise low will wrap, resulting in infinite loop + while (low < high) { + chars_[static_cast(low)] = 1; + ++low; + } + chars_[static_cast(high)] = 1; + return *this; +} + +CharacterSet +CharacterSet::complement(const char *label) const +{ + CharacterSet result((label ? label : "complement_of_some_other_set"), ""); + // negate each of our elements and add them to the result storage + std::transform(chars_.begin(), chars_.end(), result.chars_.begin(), + std::logical_not()); + return result; +} + +CharacterSet::CharacterSet(const char *label, const char * const c) : + name(label ? label: "anonymous"), + chars_(Storage(256,0)) +{ + const size_t clen = strlen(c); + for (size_t i = 0; i < clen; ++i) + add(c[i]); +} + +CharacterSet::CharacterSet(const char *label, unsigned char low, unsigned char high) : + name(label ? label: "anonymous"), + chars_(Storage(256,0)) +{ + addRange(low,high); +} + +CharacterSet::CharacterSet(const char *label, std::initializer_list> ranges) : + name(label ? label: "anonymous"), + chars_(Storage(256,0)) +{ + for (auto range: ranges) + addRange(range.first, range.second); +} + +void +CharacterSet::printChars(std::ostream &os) const +{ + for (size_t idx = 0; idx < 256; ++idx) { + if (chars_[idx]) + os << static_cast(idx); + } +} + +CharacterSet +operator+ (CharacterSet lhs, const CharacterSet &rhs) +{ + lhs += rhs; + return lhs; +} + +CharacterSet +operator- (CharacterSet lhs, const CharacterSet &rhs) +{ + lhs -= rhs; + return lhs; +} + +std::ostream& +operator <<(std::ostream &s, const CharacterSet &c) +{ + s << "CharacterSet(" << c.name << ')'; + return s; +} + +const CharacterSet +// RFC 5234 +CharacterSet::ALPHA("ALPHA", "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"), + CharacterSet::BIT("BIT","01"), + CharacterSet::CR("CR","\r"), +CharacterSet::CTL("CTL", {{0x01,0x1f},{0x7f,0x7f}}), +CharacterSet::DIGIT("DIGIT","0123456789"), +CharacterSet::DQUOTE("DQUOTE","\""), +CharacterSet::HEXDIG("HEXDIG","0123456789aAbBcCdDeEfF"), +CharacterSet::HTAB("HTAB","\t"), +CharacterSet::LF("LF","\n"), +CharacterSet::SP("SP"," "), +CharacterSet::VCHAR("VCHAR", 0x21, 0x7e), +// RFC 7230 +CharacterSet::WSP("WSP"," \t"), +CharacterSet::CTEXT("ctext", {{0x09,0x09},{0x20,0x20},{0x2a,0x5b},{0x5d,0x7e},{0x80,0xff}}), +CharacterSet::TCHAR("TCHAR","!#$%&'*+-.^_`|~0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"), +CharacterSet::SPECIAL("SPECIAL","()<>@,;:\\\"/[]?={}"), +CharacterSet::QDTEXT("QDTEXT", {{0x09,0x09},{0x20,0x21},{0x23,0x5b},{0x5d,0x7e},{0x80,0xff}}), +CharacterSet::OBSTEXT("OBSTEXT",0x80,0xff), +// RFC 7232 +CharacterSet::ETAGC("ETAGC", {{0x21,0x21},{0x23,0x7e},{0x80,0xff}}), +// RFC 7235 +CharacterSet::TOKEN68C("TOKEN68C","-._~+/0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") +; + diff -u -r -N squid-4.8/tools/helper-mux/helper-mux.8 squid-4.9/tools/helper-mux/helper-mux.8 --- squid-4.8/tools/helper-mux/helper-mux.8 2019-07-10 07:25:15.000000000 +1200 +++ squid-4.9/tools/helper-mux/helper-mux.8 2019-11-06 08:27:26.000000000 +1300 @@ -133,7 +133,7 @@ .\" ======================================================================== .\" .IX Title "HELPER-MUX 8" -.TH HELPER-MUX 8 "2019-07-09" "perl v5.28.1" "User Contributed Perl Documentation" +.TH HELPER-MUX 8 "2019-11-05" "perl v5.28.1" "User Contributed Perl Documentation" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l diff -u -r -N squid-4.8/tools/Makefile.am squid-4.9/tools/Makefile.am --- squid-4.8/tools/Makefile.am 2019-07-10 07:05:20.000000000 +1200 +++ squid-4.9/tools/Makefile.am 2019-11-06 08:14:40.000000000 +1300 @@ -37,6 +37,9 @@ Here.cc: $(top_srcdir)/src/base/Here.cc cp $(top_srcdir)/src/base/Here.cc $@ +CharacterSet.cc: $(top_srcdir)/src/base/CharacterSet.cc + cp $(top_srcdir)/src/base/CharacterSet.cc $@ + MemBuf.cc: $(top_srcdir)/src/MemBuf.cc cp $(top_srcdir)/src/MemBuf.cc $@ @@ -48,7 +51,7 @@ stub_libmem.cc: $(top_srcdir)/src/tests/stub_libmem.cc STUB.h cp $(top_srcdir)/src/tests/stub_libmem.cc $@ - + STUB.h: $(top_srcdir)/src/tests/STUB.h cp $(top_srcdir)/src/tests/STUB.h $@ @@ -57,7 +60,7 @@ # globals.cc is needed by test_tools.cc. # Neither of these should be disted from here. TESTSOURCES= test_tools.cc -CLEANFILES += test_tools.cc Here.cc MemBuf.cc stub_debug.cc time.cc stub_cbdata.cc stub_libmem.cc STUB.h +CLEANFILES += test_tools.cc Here.cc CharacterSet.cc MemBuf.cc stub_debug.cc time.cc stub_cbdata.cc stub_libmem.cc STUB.h ## Test Scripts EXTRA_DIST += helper-ok-dying.pl helper-ok.pl @@ -69,6 +72,7 @@ libexec_PROGRAMS = cachemgr$(CGIEXT) cachemgr__CGIEXT__SOURCES = cachemgr.cc \ + CharacterSet.cc \ Here.cc \ MemBuf.cc \ stub_cbdata.cc \ diff -u -r -N squid-4.8/tools/Makefile.in squid-4.9/tools/Makefile.in --- squid-4.8/tools/Makefile.in 2019-07-10 07:16:51.000000000 +1200 +++ squid-4.9/tools/Makefile.in 2019-11-06 08:19:05.000000000 +1300 @@ -166,6 +166,7 @@ am__installdirs = "$(DESTDIR)$(libexecdir)" "$(DESTDIR)$(man8dir)" PROGRAMS = $(libexec_PROGRAMS) am_cachemgr__CGIEXT__OBJECTS = cachemgr__CGIEXT_-cachemgr.$(OBJEXT) \ + cachemgr__CGIEXT_-CharacterSet.$(OBJEXT) \ cachemgr__CGIEXT_-Here.$(OBJEXT) \ cachemgr__CGIEXT_-MemBuf.$(OBJEXT) \ cachemgr__CGIEXT_-stub_cbdata.$(OBJEXT) \ @@ -207,7 +208,8 @@ DEFAULT_INCLUDES = depcomp = $(SHELL) $(top_srcdir)/cfgaux/depcomp am__maybe_remake_depfiles = depfiles -am__depfiles_remade = ./$(DEPDIR)/cachemgr__CGIEXT_-Here.Po \ +am__depfiles_remade = ./$(DEPDIR)/cachemgr__CGIEXT_-CharacterSet.Po \ + ./$(DEPDIR)/cachemgr__CGIEXT_-Here.Po \ ./$(DEPDIR)/cachemgr__CGIEXT_-MemBuf.Po \ ./$(DEPDIR)/cachemgr__CGIEXT_-cachemgr.Po \ ./$(DEPDIR)/cachemgr__CGIEXT_-stub_cbdata.Po \ @@ -781,8 +783,9 @@ DEFAULT_ERROR_DIR = $(datadir)/errors AM_CFLAGS = $(SQUID_CFLAGS) AM_CXXFLAGS = $(SQUID_CXXFLAGS) -CLEANFILES = test_tools.cc Here.cc MemBuf.cc stub_debug.cc time.cc \ - stub_cbdata.cc stub_libmem.cc STUB.h cachemgr.cgi.8 +CLEANFILES = test_tools.cc Here.cc CharacterSet.cc MemBuf.cc \ + stub_debug.cc time.cc stub_cbdata.cc stub_libmem.cc STUB.h \ + cachemgr.cgi.8 AM_CPPFLAGS = -I$(top_srcdir) -I$(top_srcdir)/include \ -I$(top_srcdir)/lib -I$(top_srcdir)/src \ -I$(top_builddir)/include $(LIBCPPUNIT_CFLAGS) $(KRB5INCS) \ @@ -822,6 +825,7 @@ TESTSOURCES = test_tools.cc DEFAULT_CACHEMGR_CONFIG = $(sysconfdir)/cachemgr.conf cachemgr__CGIEXT__SOURCES = cachemgr.cc \ + CharacterSet.cc \ Here.cc \ MemBuf.cc \ stub_cbdata.cc \ @@ -935,6 +939,7 @@ distclean-compile: -rm -f *.tab.c +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/cachemgr__CGIEXT_-CharacterSet.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/cachemgr__CGIEXT_-Here.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/cachemgr__CGIEXT_-MemBuf.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/cachemgr__CGIEXT_-cachemgr.Po@am__quote@ # am--include-marker @@ -988,6 +993,20 @@ @AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ @am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(cachemgr__CGIEXT__CXXFLAGS) $(CXXFLAGS) -c -o cachemgr__CGIEXT_-cachemgr.obj `if test -f 'cachemgr.cc'; then $(CYGPATH_W) 'cachemgr.cc'; else $(CYGPATH_W) '$(srcdir)/cachemgr.cc'; fi` +cachemgr__CGIEXT_-CharacterSet.o: CharacterSet.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(cachemgr__CGIEXT__CXXFLAGS) $(CXXFLAGS) -MT cachemgr__CGIEXT_-CharacterSet.o -MD -MP -MF $(DEPDIR)/cachemgr__CGIEXT_-CharacterSet.Tpo -c -o cachemgr__CGIEXT_-CharacterSet.o `test -f 'CharacterSet.cc' || echo '$(srcdir)/'`CharacterSet.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/cachemgr__CGIEXT_-CharacterSet.Tpo $(DEPDIR)/cachemgr__CGIEXT_-CharacterSet.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='CharacterSet.cc' object='cachemgr__CGIEXT_-CharacterSet.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(cachemgr__CGIEXT__CXXFLAGS) $(CXXFLAGS) -c -o cachemgr__CGIEXT_-CharacterSet.o `test -f 'CharacterSet.cc' || echo '$(srcdir)/'`CharacterSet.cc + +cachemgr__CGIEXT_-CharacterSet.obj: CharacterSet.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(cachemgr__CGIEXT__CXXFLAGS) $(CXXFLAGS) -MT cachemgr__CGIEXT_-CharacterSet.obj -MD -MP -MF $(DEPDIR)/cachemgr__CGIEXT_-CharacterSet.Tpo -c -o cachemgr__CGIEXT_-CharacterSet.obj `if test -f 'CharacterSet.cc'; then $(CYGPATH_W) 'CharacterSet.cc'; else $(CYGPATH_W) '$(srcdir)/CharacterSet.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/cachemgr__CGIEXT_-CharacterSet.Tpo $(DEPDIR)/cachemgr__CGIEXT_-CharacterSet.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='CharacterSet.cc' object='cachemgr__CGIEXT_-CharacterSet.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(cachemgr__CGIEXT__CXXFLAGS) $(CXXFLAGS) -c -o cachemgr__CGIEXT_-CharacterSet.obj `if test -f 'CharacterSet.cc'; then $(CYGPATH_W) 'CharacterSet.cc'; else $(CYGPATH_W) '$(srcdir)/CharacterSet.cc'; fi` + cachemgr__CGIEXT_-Here.o: Here.cc @am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(cachemgr__CGIEXT__CXXFLAGS) $(CXXFLAGS) -MT cachemgr__CGIEXT_-Here.o -MD -MP -MF $(DEPDIR)/cachemgr__CGIEXT_-Here.Tpo -c -o cachemgr__CGIEXT_-Here.o `test -f 'Here.cc' || echo '$(srcdir)/'`Here.cc @am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/cachemgr__CGIEXT_-Here.Tpo $(DEPDIR)/cachemgr__CGIEXT_-Here.Po @@ -1499,7 +1518,8 @@ clean-libtool mostlyclean-am distclean: distclean-recursive - -rm -f ./$(DEPDIR)/cachemgr__CGIEXT_-Here.Po + -rm -f ./$(DEPDIR)/cachemgr__CGIEXT_-CharacterSet.Po + -rm -f ./$(DEPDIR)/cachemgr__CGIEXT_-Here.Po -rm -f ./$(DEPDIR)/cachemgr__CGIEXT_-MemBuf.Po -rm -f ./$(DEPDIR)/cachemgr__CGIEXT_-cachemgr.Po -rm -f ./$(DEPDIR)/cachemgr__CGIEXT_-stub_cbdata.Po @@ -1552,7 +1572,8 @@ installcheck-am: maintainer-clean: maintainer-clean-recursive - -rm -f ./$(DEPDIR)/cachemgr__CGIEXT_-Here.Po + -rm -f ./$(DEPDIR)/cachemgr__CGIEXT_-CharacterSet.Po + -rm -f ./$(DEPDIR)/cachemgr__CGIEXT_-Here.Po -rm -f ./$(DEPDIR)/cachemgr__CGIEXT_-MemBuf.Po -rm -f ./$(DEPDIR)/cachemgr__CGIEXT_-cachemgr.Po -rm -f ./$(DEPDIR)/cachemgr__CGIEXT_-stub_cbdata.Po @@ -1615,6 +1636,9 @@ Here.cc: $(top_srcdir)/src/base/Here.cc cp $(top_srcdir)/src/base/Here.cc $@ +CharacterSet.cc: $(top_srcdir)/src/base/CharacterSet.cc + cp $(top_srcdir)/src/base/CharacterSet.cc $@ + MemBuf.cc: $(top_srcdir)/src/MemBuf.cc cp $(top_srcdir)/src/MemBuf.cc $@