diff -ruN squid-2.6.STABLE2/ChangeLog squid-2.6.STABLE3/ChangeLog
--- squid-2.6.STABLE2/ChangeLog Sun Jul 30 20:53:23 2006
+++ squid-2.6.STABLE3/ChangeLog Fri Aug 18 15:31:01 2006
@@ -1,3 +1,32 @@
+Changes to squid-2.5.STABLE3 (Aug 18 2006)
+
+ - Bug #1577: assertion failed "fm->max_n_files <= (1 << 24)" on
+ very large cache_dir. Limit number of objects stored to slightly
+ less to avoid this.
+ - Bug #1705: Correct error message on invalid time weekday specification
+ - Don't attempt to guess netmask in src/dst acl specifications
+ if none was provided. Assume it's an IP even if it ends in 0
+ - Bug #1665: log_format %ue, %us tags for external or ssl user id
+ - Bug #1707: delay pools often ignored the set limit
+ - Bug #1716: Support for recent OpenSSL 0.9.7 versions
+ (0.9.8 always worked)
+ - COSS fixes and performance improvements
+ - Memory leak when reading configuration files with overlapping
+ ACL data where squid -k parse complains.
+ - Memory leak related to pinned connections
+ - Show include acls unexpanded in cachemgr configuration dumps
+ - Fixed WARNING defer handler for HTTP Socket does not call commDeferFD
+ - Bug #1304: Downloads may hang when using the cache_dir max-size option
+ - Optimization of network I/O
+ - Bug #1730: make problem with --enable-follow-x-forwarded-for on Solaris
+ - Fixed a memory leak on certain invalid requests
+ - Bug #1733: ERR_CANNOT_FORWARD Portuguese translation update
+ - Bug #582: ntlm fake_auth not handles non-ascii login names
+ - New startup message indicating the type of event loop used
+ - Bug #1602: TCP fallback on truncated DNS responses
+ - Bug #1667: assertion failed: store.c:1081: "e->store_status == STORE_PENDING"
+ - Bug #1723: cachemgr now works in accelerator mode
+
Changes to squid-2.6.STABLE2 (31 Jul 2006)
- WCCP2 doesn't update statCounter.syscalls.sock.sendtos counter.
diff -ruN squid-2.6.STABLE2/RELEASENOTES.html squid-2.6.STABLE3/RELEASENOTES.html
--- squid-2.6.STABLE2/RELEASENOTES.html Sun Jul 30 20:54:08 2006
+++ squid-2.6.STABLE3/RELEASENOTES.html Fri Aug 18 15:33:10 2006
@@ -7,7 +7,7 @@
Squid 2.6 release notes
-Squid Developers
$Id: release-2.6.html,v 1.27 2006/07/31 00:36:05 hno Exp $
+Squid Developers
$Id: release-2.6.html,v 1.28 2006/08/18 21:30:10 hno Exp $
This document contains the release notes for version 2.6 of Squid.
Squid is a WWW Cache application developed by the Web Caching community.
@@ -33,6 +33,9 @@
+
+
+
@@ -335,6 +338,31 @@
new minimum_expiry_time squid.conf directive backported from Squid-3
Bug #1703: Wrong default path to the diskd helper causing hangs at 100% CPU
Bug #1685: Crashes or other odd results after storeSwapMetaUnpack: errors
+a number of other minor and cosmetic bugfixes. See the list of
+squid-2.6 changes and the
+ChangeLog file for details.
+
+
+
+
+
+
+
+
+- src/dst acl parsing changed to not attempt to guess a netmask
+if none was specified. Instead assume it's an IP address and not
+a network even if it ends in 0
+- Several memory leaks plugged
+- Delay pools now work again (broken in 2.6.STABLE1 & 2)
+- New log_format %ue and %us tags for external acl or ssl user id
+- COSS fixes and performance improvements
+- Include acl's is now shown in their original form in cachemgr configuration dumps.
+- ntlm fake_auth finally handles non-ascii user names
+- TCP fallback on truncated DNS responses, making the internal
+DNS client complete.
+- Downloads could hang when using the cache_dir max-size option
+- Fixed some assertion failures and segmentation faults
+- Some small optimizations to reduce CPU usage
- a number of other minor and cosmetic bugfixes. See the list of
squid-2.6 changes and the
ChangeLog file for details.
diff -ruN squid-2.6.STABLE2/configure squid-2.6.STABLE3/configure
--- squid-2.6.STABLE2/configure Sun Jul 30 20:53:51 2006
+++ squid-2.6.STABLE3/configure Fri Aug 18 15:32:54 2006
@@ -1,7 +1,7 @@
#! /bin/sh
-# From configure.in Revision: 1.390 .
+# From configure.in Revision: 1.392 .
# Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.59 for Squid Web Proxy 2.6.STABLE2.
+# Generated by GNU Autoconf 2.59 for Squid Web Proxy 2.6.STABLE3.
#
# Report bugs to .
#
@@ -270,8 +270,8 @@
# Identity of this package.
PACKAGE_NAME='Squid Web Proxy'
PACKAGE_TARNAME='squid'
-PACKAGE_VERSION='2.6.STABLE2'
-PACKAGE_STRING='Squid Web Proxy 2.6.STABLE2'
+PACKAGE_VERSION='2.6.STABLE3'
+PACKAGE_STRING='Squid Web Proxy 2.6.STABLE3'
PACKAGE_BUGREPORT='http://www.squid-cache.org/bugs/'
ac_default_prefix=/usr/local/squid
@@ -781,7 +781,7 @@
# Omit some internal or obsolete options to make the list less imposing.
# This message is too long to be a string in the A/UX 3.1 sh.
cat <<_ACEOF
-\`configure' configures Squid Web Proxy 2.6.STABLE2 to adapt to many kinds of systems.
+\`configure' configures Squid Web Proxy 2.6.STABLE3 to adapt to many kinds of systems.
Usage: $0 [OPTION]... [VAR=VALUE]...
@@ -847,7 +847,7 @@
if test -n "$ac_init_help"; then
case $ac_init_help in
- short | recursive ) echo "Configuration of Squid Web Proxy 2.6.STABLE2:";;
+ short | recursive ) echo "Configuration of Squid Web Proxy 2.6.STABLE3:";;
esac
cat <<\_ACEOF
@@ -1155,7 +1155,7 @@
test -n "$ac_init_help" && exit 0
if $ac_init_version; then
cat <<\_ACEOF
-Squid Web Proxy configure 2.6.STABLE2
+Squid Web Proxy configure 2.6.STABLE3
generated by GNU Autoconf 2.59
Copyright (C) 2003 Free Software Foundation, Inc.
@@ -1169,7 +1169,7 @@
This file contains any messages produced by compilers while
running configure, to aid debugging if configure makes a mistake.
-It was created by Squid Web Proxy $as_me 2.6.STABLE2, which was
+It was created by Squid Web Proxy $as_me 2.6.STABLE3, which was
generated by GNU Autoconf 2.59. Invocation command line was
$ $0 $@
@@ -1815,7 +1815,7 @@
# Define the identity of the package.
PACKAGE='squid'
- VERSION='2.6.STABLE2'
+ VERSION='2.6.STABLE3'
cat >>confdefs.h <<_ACEOF
@@ -4632,8 +4632,11 @@
USE_DNSSERVER_FALSE=
fi
- case "$host_os" in
- cygwin|cygwin32)
+ case "$host" in
+ *-solaris*)
+ LIBS="$LIBS -lresolv"
+ ;;
+ *-cygwin*)
LIBS="$LIBS -lresolv"
;;
esac
@@ -5634,7 +5637,132 @@
#define FOLLOW_X_FORWARDED_FOR 1
_ACEOF
- fi
+ echo "$as_me:$LINENO: checking for library containing inet_aton" >&5
+echo $ECHO_N "checking for library containing inet_aton... $ECHO_C" >&6
+if test "${ac_cv_search_inet_aton+set}" = set; then
+ echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+ ac_func_search_save_LIBS=$LIBS
+ac_cv_search_inet_aton=no
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h. */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h. */
+
+/* Override any gcc2 internal prototype to avoid an error. */
+#ifdef __cplusplus
+extern "C"
+#endif
+/* We use char because int might match the return type of a gcc2
+ builtin and then its argument prototype would still apply. */
+char inet_aton ();
+int
+main ()
+{
+inet_aton ();
+ ;
+ return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+ (eval $ac_link) 2>conftest.er1
+ ac_status=$?
+ grep -v '^ *+' conftest.er1 >conftest.err
+ rm -f conftest.er1
+ cat conftest.err >&5
+ echo "$as_me:$LINENO: \$? = $ac_status" >&5
+ (exit $ac_status); } &&
+ { ac_try='test -z "$ac_c_werror_flag"
+ || test ! -s conftest.err'
+ { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+ (eval $ac_try) 2>&5
+ ac_status=$?
+ echo "$as_me:$LINENO: \$? = $ac_status" >&5
+ (exit $ac_status); }; } &&
+ { ac_try='test -s conftest$ac_exeext'
+ { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+ (eval $ac_try) 2>&5
+ ac_status=$?
+ echo "$as_me:$LINENO: \$? = $ac_status" >&5
+ (exit $ac_status); }; }; then
+ ac_cv_search_inet_aton="none required"
+else
+ echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+if test "$ac_cv_search_inet_aton" = no; then
+ for ac_lib in resolv; do
+ LIBS="-l$ac_lib $ac_func_search_save_LIBS"
+ cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h. */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h. */
+
+/* Override any gcc2 internal prototype to avoid an error. */
+#ifdef __cplusplus
+extern "C"
+#endif
+/* We use char because int might match the return type of a gcc2
+ builtin and then its argument prototype would still apply. */
+char inet_aton ();
+int
+main ()
+{
+inet_aton ();
+ ;
+ return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+ (eval $ac_link) 2>conftest.er1
+ ac_status=$?
+ grep -v '^ *+' conftest.er1 >conftest.err
+ rm -f conftest.er1
+ cat conftest.err >&5
+ echo "$as_me:$LINENO: \$? = $ac_status" >&5
+ (exit $ac_status); } &&
+ { ac_try='test -z "$ac_c_werror_flag"
+ || test ! -s conftest.err'
+ { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+ (eval $ac_try) 2>&5
+ ac_status=$?
+ echo "$as_me:$LINENO: \$? = $ac_status" >&5
+ (exit $ac_status); }; } &&
+ { ac_try='test -s conftest$ac_exeext'
+ { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+ (eval $ac_try) 2>&5
+ ac_status=$?
+ echo "$as_me:$LINENO: \$? = $ac_status" >&5
+ (exit $ac_status); }; }; then
+ ac_cv_search_inet_aton="-l$ac_lib"
+break
+else
+ echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+ done
+fi
+LIBS=$ac_func_search_save_LIBS
+fi
+echo "$as_me:$LINENO: result: $ac_cv_search_inet_aton" >&5
+echo "${ECHO_T}$ac_cv_search_inet_aton" >&6
+if test "$ac_cv_search_inet_aton" != no; then
+ test "$ac_cv_search_inet_aton" = "none required" || LIBS="$ac_cv_search_inet_aton $LIBS"
+
+fi
+ fi
fi;
@@ -18163,7 +18291,7 @@
} >&5
cat >&5 <<_CSEOF
-This file was extended by Squid Web Proxy $as_me 2.6.STABLE2, which was
+This file was extended by Squid Web Proxy $as_me 2.6.STABLE3, which was
generated by GNU Autoconf 2.59. Invocation command line was
CONFIG_FILES = $CONFIG_FILES
@@ -18226,7 +18354,7 @@
cat >>$CONFIG_STATUS <<_ACEOF
ac_cs_version="\\
-Squid Web Proxy config.status 2.6.STABLE2
+Squid Web Proxy config.status 2.6.STABLE3
configured by $0, generated by GNU Autoconf 2.59,
with options \\"`echo "$ac_configure_args" | sed 's/[\\""\`\$]/\\\\&/g'`\\"
diff -ruN squid-2.6.STABLE2/configure.in squid-2.6.STABLE3/configure.in
--- squid-2.6.STABLE2/configure.in Sun Jul 30 20:53:51 2006
+++ squid-2.6.STABLE3/configure.in Fri Aug 18 15:32:54 2006
@@ -1,16 +1,16 @@
dnl
dnl Configuration input file for Squid
dnl
-dnl $Id: configure.in,v 1.390 2006/07/31 02:38:45 hno Exp $
+dnl $Id: configure.in,v 1.392 2006/08/18 21:31:01 hno Exp $
dnl
dnl
dnl
-AC_INIT(Squid Web Proxy, 2.6.STABLE2, http://www.squid-cache.org/bugs/, squid)
+AC_INIT(Squid Web Proxy, 2.6.STABLE3, http://www.squid-cache.org/bugs/, squid)
AC_PREREQ(2.52)
AM_CONFIG_HEADER(include/autoconf.h)
AC_CONFIG_AUX_DIR(cfgaux)
AM_INIT_AUTOMAKE
-AC_REVISION($Revision: 1.390 $)dnl
+AC_REVISION($Revision: 1.392 $)dnl
AC_PREFIX_DEFAULT(/usr/local/squid)
AM_MAINTAINER_MODE
@@ -1100,9 +1100,12 @@
AC_DEFINE(USE_DNSSERVERS, 1, [If --disable-internal-dns was given to configure, then we'll use
the dnsserver processes instead.])
AM_CONDITIONAL(USE_DNSSERVER, true)
- dnl Cygwin needs -lresolv when using --disable-internal-dns
- case "$host_os" in
- cygwin|cygwin32)
+ dnl Some platforms need -lresolv when using --disable-internal-dns
+ case "$host" in
+ *-solaris*)
+ LIBS="$LIBS -lresolv"
+ ;;
+ *-cygwin*)
LIBS="$LIBS -lresolv"
;;
esac
@@ -1443,6 +1446,7 @@
[ if test "$enableval" = "yes" ; then
echo "follow X-Forwarded-For enabled"
AC_DEFINE(FOLLOW_X_FORWARDED_FOR, 1, [Enable following X-Forwarded-For headers])
+ AC_SEARCH_LIBS(inet_aton, resolv) dnl some systems don't have this in libc
fi
])
diff -ruN squid-2.6.STABLE2/errors/Portuguese/ERR_CANNOT_FORWARD squid-2.6.STABLE3/errors/Portuguese/ERR_CANNOT_FORWARD
--- squid-2.6.STABLE2/errors/Portuguese/ERR_CANNOT_FORWARD Wed Aug 28 14:54:37 2002
+++ squid-2.6.STABLE3/errors/Portuguese/ERR_CANNOT_FORWARD Mon Aug 14 01:42:36 2006
@@ -16,12 +16,12 @@
-
-Cache incapaz de encaminhar esta requsição neste momento.
+Cache incapaz de encaminhar esta requisição neste momento.
-Esta requsição não pode ser encaminhada para o servidor ou
+Esta requisição não pode ser encaminhada para o servidor ou
para qualquer outro cache. A causa mais provável para este erro é:
- O administrador do cache não permite que este cache faça
diff -ruN squid-2.6.STABLE2/helpers/ntlm_auth/fakeauth/fakeauth_auth.c squid-2.6.STABLE3/helpers/ntlm_auth/fakeauth/fakeauth_auth.c
--- squid-2.6.STABLE2/helpers/ntlm_auth/fakeauth/fakeauth_auth.c Sun Oct 23 09:20:48 2005
+++ squid-2.6.STABLE3/helpers/ntlm_auth/fakeauth/fakeauth_auth.c Tue Aug 15 18:54:15 2006
@@ -10,13 +10,18 @@
* This code gets the username and returns it. No validation is done.
* and by the way: it is a complete patch-up. Use the "real thing" NTLMSSP
* if you can.
+ *
+ * Revised by Guido Serassio:
+ *
+ * - Added negotiation of UNICODE char support
+ * - More detailed debugging info
+ *
*/
#include "config.h"
#include "squid_types.h"
#include "ntlmauth.h"
-#include "ntlm.h"
#include "util.h"
#include
@@ -41,15 +46,83 @@
#if HAVE_ASSERT_H
#include
#endif
-
-#if 0
-#define NTLM_STATIC_CHALLENGE "deadbeef"
+#if HAVE_GETOPT_H
+#include
#endif
-static char *authenticate_ntlm_domain = "LIFELESSWKS";
+#include "ntlm.h"
+
+#define BUFFER_SIZE 10240
+
+char *authenticate_ntlm_domain = "WORKGROUP";
+int debug_enabled = 0;
+int NTLM_packet_debug_enabled = 0;
/* NTLM authentication by ad@interlude.eu.org - 07/1999 */
/* XXX this is not done cleanly... */
+void
+hex_dump(void *data, int size)
+{
+ /* dumps size bytes of *data to stdout. Looks like:
+ * [0000] 75 6E 6B 6E 6F 77 6E 20
+ * 30 FF 00 00 00 00 39 00 unknown 0.....9.
+ * (in a single line of course)
+ */
+
+ if (!data)
+ return;
+
+ if (debug_enabled) {
+ unsigned char *p = data;
+ unsigned char c;
+ int n;
+ char bytestr[4] =
+ {0};
+ char addrstr[10] =
+ {0};
+ char hexstr[16 * 3 + 5] =
+ {0};
+ char charstr[16 * 1 + 5] =
+ {0};
+ for (n = 1; n <= size; n++) {
+ if (n % 16 == 1) {
+ /* store address for this line */
+ snprintf(addrstr, sizeof(addrstr), "%.4x",
+ (unsigned int)(p - (unsigned char *)data));
+ }
+ c = *p;
+ if (isalnum(c) == 0) {
+ c = '.';
+ }
+ /* store hex str (for left side) */
+ snprintf(bytestr, sizeof(bytestr), "%02X ", *p);
+ strncat(hexstr, bytestr, sizeof(hexstr) - strlen(hexstr) - 1);
+
+ /* store char str (for right side) */
+ snprintf(bytestr, sizeof(bytestr), "%c", c);
+ strncat(charstr, bytestr, sizeof(charstr) - strlen(charstr) - 1);
+
+ if (n % 16 == 0) {
+ /* line completed */
+ fprintf(stderr, "[%4.4s] %-50.50s %s\n", addrstr, hexstr, charstr);
+ hexstr[0] = 0;
+ charstr[0] = 0;
+ } else if (n % 8 == 0) {
+ /* half line: add whitespaces */
+ strncat(hexstr, " ", sizeof(hexstr) - strlen(hexstr) - 1);
+ strncat(charstr, " ", sizeof(charstr) - strlen(charstr) - 1);
+ }
+ p++; /* next byte */
+ }
+
+ if (strlen(hexstr) > 0) {
+ /* print rest of buffer if not empty */
+ fprintf(stderr, "[%4.4s] %-50.50s %s\n", addrstr, hexstr, charstr);
+ }
+ }
+}
+
+
/* makes a null-terminated string lower-case. Changes CONTENTS! */
static void
lc(char *string)
@@ -68,18 +141,23 @@
* challenge strings can be guaranteed to be poor at best.
*/
void
-ntlmMakeChallenge(struct ntlm_challenge *chal)
+ntlmMakeChallenge(struct ntlm_challenge *chal, int32_t flags)
{
-#ifndef NTLM_STATIC_CHALLENGE
static unsigned hash;
int r;
-#endif
char *d;
int i;
+ debug("ntlmMakeChallenge: flg %08x\n", flags);
+
memset(chal, 0, sizeof(*chal));
memcpy(chal->hdr.signature, "NTLMSSP", 8);
- chal->flags = WSWAP(0x00018206);
+ chal->flags = WSWAP(CHALLENGE_TARGET_IS_DOMAIN |
+ NEGOTIATE_ALWAYS_SIGN |
+ NEGOTIATE_USE_NTLM |
+ NEGOTIATE_REQUEST_TARGET |
+ (NEGOTIATE_UNICODE & flags ? NEGOTIATE_UNICODE : NEGOTIATE_ASCII)
+ );
chal->hdr.type = WSWAP(NTLM_CHALLENGE);
chal->unknown[6] = SSWAP(0x003a);
@@ -93,9 +171,6 @@
chal->target.maxlen = SSWAP(i);
chal->target.len = chal->target.maxlen;
-#ifdef NTLM_STATIC_CHALLENGE
- memcpy(chal->challenge, NTLM_STATIC_CHALLENGE, 8);
-#else
r = (int) rand();
r = (hash ^ r) + r;
@@ -105,7 +180,6 @@
}
hash = r;
-#endif
}
/*
@@ -155,29 +229,28 @@
fprintf(stderr, "ntlmGetString: insane: l:%d o:%d\n", l, o);
return (NULL);
}
- if ((flags & 2) == 0) {
+ if ((flags & NEGOTIATE_ASCII) == 0) {
/* UNICODE string */
s = (u_short *) ((char *) hdr + o);
d = buf;
for (l >>= 1; l; s++, l--) {
c = SSWAP(*s);
- if (c > 254 || c == '\0' || !isprint(c)) {
+ if (c > 254 || c == '\0') {
fprintf(stderr, "ntlmGetString: bad uni: %04x\n", c);
return (NULL);
}
*d++ = c;
- fprintf(stderr, "ntlmGetString: conv: '%c'\n", c);
}
*d = 0;
} else {
- /* ASCII string */
+ /* ASCII/OEM string */
sc = (char *) hdr + o;
d = buf;
for (; l; l--) {
- if (*sc == '\0' || !isprint((int)(unsigned char)*sc)) {
+ if (*sc == '\0' || !isprint((int) (unsigned char) *sc)) {
fprintf(stderr, "ntlmGetString: bad ascii: %04x\n", *sc);
return (NULL);
}
@@ -203,32 +276,27 @@
if (!buf)
return 1;
origbuf = buf;
- assert (0 == ntlmCheckHeader(&auth->hdr, NTLM_AUTHENTICATE));
-
-#if DEBUG_FAKEAUTH
- fprintf(stderr,"ntlmDecodeAuth: size of %d\n", size);
- fprintf(stderr,"ntlmDecodeAuth: flg %08x\n", auth->flags);
- fprintf(stderr,"ntlmDecodeAuth: usr o(%d) l(%d)\n", auth->user.offset,
- auth->user.len);
-#endif
+ if (ntlmCheckHeader(&auth->hdr, NTLM_AUTHENTICATE)) {
+ fprintf(stderr, "ntlmDecodeAuth: header check fails\n");
+ return -1;
+ }
+ debug("ntlmDecodeAuth: size of %d\n", (int)size);
+ debug("ntlmDecodeAuth: flg %08x\n", auth->flags);
+ debug("ntlmDecodeAuth: usr o(%d) l(%d)\n", auth->user.offset, auth->user.len);
- if ((p = ntlmGetString(&auth->hdr, &auth->domain, 2)) == NULL)
+ if ((p = ntlmGetString(&auth->hdr, &auth->domain, auth->flags)) == NULL)
p = authenticate_ntlm_domain;
-#if DEBUG_FAKEAUTH
- fprintf(stderr,"ntlmDecodeAuth: Domain '%s'.\n",p);
-#endif
+ debug("ntlmDecodeAuth: Domain '%s'.\n", p);
if ((s = strlen(p) + 1) >= size)
return 1;
strcpy(buf, p);
-#if DEBUG_FAKEAUTH
- fprintf(stdout,"ntlmDecodeAuth: Domain '%s'.\n",buf);
-#endif
+ debug("ntlmDecodeAuth: Domain '%s'.\n", buf);
size -= s;
buf += (s - 1);
*buf++ = '\\'; /* Using \ is more consistent with MS-proxy */
- p = ntlmGetString(&auth->hdr, &auth->user, 2);
+ p = ntlmGetString(&auth->hdr, &auth->user, auth->flags);
if (NULL == p)
return 1;
if ((s = strlen(p) + 1) >= size)
@@ -238,84 +306,128 @@
*buf++ = '\0';
size -= s;
-#if DEBUG_FAKEAUTH
- fprintf(stderr, "ntlmDecodeAuth: user: %s%s\n",origbuf, p);
-#endif
+ debug("ntlmDecodeAuth: user: %s%s\n", origbuf, p);
return 0;
}
+/*
+ * options:
+ * -d enable debugging.
+ * -v enable verbose NTLM packet debugging.
+ * -l if specified, changes behavior on failures to last-ditch.
+ */
+char *my_program_name = NULL;
+
+void
+usage()
+{
+ fprintf(stderr,
+ "Usage: %s [-d] [-v] [-h]\n"
+ " -d enable debugging.\n"
+ " -v enable verbose NTLM packet debugging.\n"
+ " -h this message\n\n",
+ my_program_name);
+}
+
+
+void
+process_options(int argc, char *argv[])
+{
+ int opt, had_error = 0;
+
+ opterr = 0;
+ while (-1 != (opt = getopt(argc, argv, "hdv"))) {
+ switch (opt) {
+ case 'd':
+ debug_enabled = 1;
+ break;
+ case 'v':
+ debug_enabled = 1;
+ NTLM_packet_debug_enabled = 1;
+ break;
+ case 'h':
+ usage();
+ exit(0);
+ case '?':
+ opt = optopt;
+ /* fall thru to default */
+ default:
+ fprintf(stderr, "unknown option: -%c. Exiting\n", opt);
+ usage();
+ had_error = 1;
+ }
+ }
+ if (had_error)
+ exit(1);
+}
+
int
main(int argc, char *argv[])
{
- char buf[256];
- char user[256];
- char *p;
- char *cleartext = NULL;
+ char buf[BUFFER_SIZE];
+ char user[256], *p, *decoded = NULL;
struct ntlm_challenge chal;
+ struct ntlm_negotiate *nego;
+ char helper_command[3];
int len;
char *data = NULL;
setbuf(stdout, NULL);
- while (fgets(buf, 256, stdin) != NULL) {
+ setbuf(stderr, NULL);
+
+ my_program_name = argv[0];
+
+ process_options(argc, argv);
+
+ debug("%s build " __DATE__ ", " __TIME__ " starting up...\n", my_program_name);
+
+ while (fgets(buf, BUFFER_SIZE, stdin) != NULL) {
memset(user, '\0', sizeof(user)); /* no usercode */
if ((p = strchr(buf, '\n')) != NULL)
*p = '\0'; /* strip \n */
-#if defined(NTLMHELPPROTOCOLV3) || !defined(NTLMHELPPROTOCOLV2)
+ if ((strlen(buf) > 3) && NTLM_packet_debug_enabled) {
+ decoded = base64_decode(buf + 3);
+ strncpy(helper_command, buf, 2);
+ helper_command[2] = '\0';
+ debug("Got '%s' from Squid with data:\n", helper_command);
+ hex_dump(decoded, ((strlen(buf) - 3) * 3) / 4);
+ } else
+ debug("Got '%s' from Squid\n", buf);
+
if (strncasecmp(buf, "YR", 2) == 0) {
- ntlmMakeChallenge(&chal);
+ if (strlen(buf) > 3) {
+ nego = (struct ntlm_negotiate *) decoded;
+ ntlmMakeChallenge(&chal, nego->flags);
+ } else
+ ntlmMakeChallenge(&chal, NEGOTIATE_ASCII);
len =
sizeof(chal) - sizeof(chal.pad) +
SSWAP(chal.target.maxlen);
data = (char *) base64_encode_bin((char *) &chal, len);
- printf("TT %s\n", data);
+ if (NTLM_packet_debug_enabled) {
+ printf("TT %s\n", data);
+ decoded = base64_decode(data);
+ debug("sending 'TT' to squid with data:\n");
+ hex_dump(decoded, (strlen(data) * 3) / 4);
+ } else
+ SEND2("TT %s", data);
} else if (strncasecmp(buf, "KK ", 3) == 0) {
- cleartext = (char *) uudecode(buf + 3);
- if (!ntlmCheckHeader((ntlmhdr *) cleartext, NTLM_AUTHENTICATE)) {
- if (!ntlmDecodeAuth((struct ntlm_authenticate *) cleartext, user, 256)) {
- lc(user);
- printf("AF %s\n", user);
- } else {
- lc(user);
- printf("NA invalid credentials, user=%s\n", user);
- }
- } else {
- lc(user);
- printf("BH wrong packet type! user=%s\n", user);
- }
- }
-#endif
-#ifdef NTLMHELPPROTOCOLV2
-/* V2 of the protocol */
- if (strncasecmp(buf, "RESET", 5) == 0) {
- printf("RESET OK\n");
- } else {
- cleartext = (char *) uudecode(buf);
- if (!ntlmCheckHeader((struct ntlmhdr *) cleartext, NTLM_NEGOTIATE)) {
- ntlmMakeChallenge(&chal);
- len =
- sizeof(chal) - sizeof(chal.pad) +
- SSWAP(chal.target.maxlen);
- data = (char *) base64_encode_bin((char *) &chal, len);
- printf("CH %s\n", data);
- } else if (!ntlmCheckHeader ((struct ntlmhdr *) cleartext, NTLM_AUTHENTICATE)) {
- if (!ntlmDecodeAuth ((struct ntlm_authenticate *) cleartext, user, 256)) {
+ if (!ntlmCheckHeader((ntlmhdr *) decoded, NTLM_AUTHENTICATE)) {
+ if (!ntlmDecodeAuth((struct ntlm_authenticate *) decoded, user, 256)) {
lc(user);
- printf("OK %s\n", user);
+ SEND2("AF %s", user);
} else {
lc(user);
- printf("ERR %s\n", user);
+ SEND2("NA invalid credentials, user=%s", user);
}
} else {
lc(user);
- printf("ERR %s\n", user);
+ SEND2("BH wrong packet type! user=%s", user);
}
}
-#endif /*v2 */
- free(cleartext);
- cleartext = NULL;
}
exit(0);
}
diff -ruN squid-2.6.STABLE2/helpers/ntlm_auth/fakeauth/ntlm.h squid-2.6.STABLE3/helpers/ntlm_auth/fakeauth/ntlm.h
--- squid-2.6.STABLE2/helpers/ntlm_auth/fakeauth/ntlm.h Wed Aug 29 08:57:38 2001
+++ squid-2.6.STABLE3/helpers/ntlm_auth/fakeauth/ntlm.h Wed Aug 16 12:37:42 2006
@@ -1,5 +1,5 @@
/*
- * $Id: ntlm.h,v 1.4 2001/08/29 14:57:38 robertc Exp $
+ * $Id: ntlm.h,v 1.6 2006/08/16 18:37:42 serassio Exp $
*
* AUTHOR: Andrew Doran
*
@@ -50,9 +50,6 @@
#endif
/* NTLM request types that we know about */
-#define NTLM_NEGOTIATE 1
-#define NTLM_CHALLENGE 2
-#define NTLM_AUTHENTICATE 3
#define NTLM_ANY 0
/* Negotiation request sent by client */
@@ -88,9 +85,83 @@
};
char *ntlmGetString(ntlmhdr * hdr, strhdr * str, int flags);
-void ntlmMakeChallenge(struct ntlm_challenge *chal);
+void ntlmMakeChallenge(struct ntlm_challenge *chal, int32_t flags);
int ntlmCheckHeader(ntlmhdr * hdr, int type);
int ntlmCheckNegotiation(struct ntlm_negotiate *neg);
int ntlmAuthenticate(struct ntlm_authenticate *neg);
+
+#define safe_free(x) if (x) { free(x); x = NULL; }
+
+#undef debug
+
+/************* CONFIGURATION ***************/
+/*
+ * define this if you want debugging
+ */
+#ifndef DEBUG
+#define DEBUG
+#endif
+
+#define FAIL_DEBUG 0
+
+/************* END CONFIGURATION ***************/
+
+#include
+
+extern int debug_enabled;
+#if FAIL_DEBUG
+extern int fail_debug_enabled;
+#endif
+
+/* Debugging stuff */
+
+#ifdef __GNUC__ /* this is really a gcc-ism */
+#ifdef DEBUG
+#include
+#include
+static char *__foo;
+#define debug(X...) if (debug_enabled) { \
+ fprintf(stderr,"ntlm-auth[%d](%s:%d): ", (int)getpid(), \
+ ((__foo=strrchr(__FILE__,'/'))==NULL?__FILE__:__foo+1),\
+ __LINE__);\
+ fprintf(stderr,X); }
+#else /* DEBUG */
+#define debug(X...) /* */
+#endif /* DEBUG */
+#else /* __GNUC__ */
+static void
+debug(char *format,...)
+{
+#ifdef DEBUG
+#ifdef _SQUID_MSWIN_
+#if FAIL_DEBUG
+ if (debug_enabled || fail_debug_enabled) {
+#else
+ if (debug_enabled) {
+#endif
+ va_list args;
+
+ va_start(args, format);
+ fprintf(stderr, "ntlm-auth[%d]: ", getpid());
+ vfprintf(stderr, format, args);
+ va_end(args);
+#if FAIL_DEBUG
+ fail_debug_enabled = 0;
+#endif
+ }
+#endif /* _SQUID_MSWIN_ */
+#endif /* DEBUG */
+}
+#endif /* __GNUC__ */
+
+
+/* A couple of harmless helper macros */
+#define SEND(X) debug("sending '%s' to squid\n",X); printf(X "\n");
+#ifdef __GNUC__
+#define SEND2(X,Y...) debug("sending '" X "' to squid\n",Y); printf(X "\n",Y);
+#else
+/* no gcc, no debugging. varargs macros are a gcc extension */
+#define SEND2(X,Y) debug("sending '" X "' to squid\n",Y); printf(X "\n",Y);
+#endif
#endif /* _NTLM_H_ */
diff -ruN squid-2.6.STABLE2/include/version.h squid-2.6.STABLE3/include/version.h
--- squid-2.6.STABLE2/include/version.h Sun Jul 30 20:53:51 2006
+++ squid-2.6.STABLE3/include/version.h Fri Aug 18 15:32:54 2006
@@ -9,5 +9,5 @@
*/
#ifndef SQUID_RELEASE_TIME
-#define SQUID_RELEASE_TIME 1154314429
+#define SQUID_RELEASE_TIME 1155936771
#endif
diff -ruN squid-2.6.STABLE2/lib/splay.c squid-2.6.STABLE3/lib/splay.c
--- squid-2.6.STABLE2/lib/splay.c Tue May 17 10:56:36 2005
+++ squid-2.6.STABLE3/lib/splay.c Sun Aug 6 12:40:59 2006
@@ -1,5 +1,5 @@
/*
- * $Id: splay.c,v 1.15 2005/05/17 16:56:36 hno Exp $
+ * $Id: splay.c,v 1.16 2006/08/06 18:40:59 hno Exp $
*/
#include "config.h"
@@ -24,11 +24,11 @@
{
splayNode *new = xcalloc(sizeof(splayNode), 1);
new->data = data;
+ top = splay_splay(data, top, compare);
if (top == NULL) {
new->left = new->right = NULL;
return new;
}
- top = splay_splay(data, top, compare);
if (splayLastResult < 0) {
new->left = top->left;
new->right = top;
diff -ruN squid-2.6.STABLE2/src/MemBuf.c squid-2.6.STABLE3/src/MemBuf.c
--- squid-2.6.STABLE2/src/MemBuf.c Sat May 20 15:51:49 2006
+++ squid-2.6.STABLE3/src/MemBuf.c Tue Aug 15 18:27:20 2006
@@ -1,6 +1,6 @@
/*
- * $Id: MemBuf.c,v 1.33 2006/05/20 21:51:49 hno Exp $
+ * $Id: MemBuf.c,v 1.34 2006/08/16 00:27:20 hno Exp $
*
* DEBUG: section 59 auto-growing Memory Buffer with printf
* AUTHOR: Alex Rousskov
@@ -191,7 +191,7 @@
/* calls memcpy, appends exactly size bytes, extends buffer if needed */
void
-memBufAppend(MemBuf * mb, const char *buf, int sz)
+memBufAppend(MemBuf * mb, const void *buf, int sz)
{
assert(mb && buf && sz >= 0);
assert(mb->buf);
diff -ruN squid-2.6.STABLE2/src/Packer.c squid-2.6.STABLE3/src/Packer.c
--- squid-2.6.STABLE2/src/Packer.c Sun Oct 23 09:20:52 2005
+++ squid-2.6.STABLE3/src/Packer.c Tue Aug 15 18:54:16 2006
@@ -1,6 +1,6 @@
/*
- * $Id: Packer.c,v 1.15 2005/10/23 15:20:52 hno Exp $
+ * $Id: Packer.c,v 1.16 2006/08/16 00:54:16 hno Exp $
*
* DEBUG: section 60 Packer: A uniform interface to store-like modules
* AUTHOR: Alex Rousskov
@@ -93,7 +93,7 @@
/* append()'s */
static void (*const store_append) (StoreEntry *, const char *, int) = &storeAppend;
-static void (*const memBuf_append) (MemBuf *, const char *, int) = &memBufAppend;
+static void (*const memBuf_append) (MemBuf *, const void *, int) = &memBufAppend;
/* vprintf()'s */
static void (*const store_vprintf) (StoreEntry *, const char *, va_list ap) = &storeAppendVPrintf;
diff -ruN squid-2.6.STABLE2/src/access_log.c squid-2.6.STABLE3/src/access_log.c
--- squid-2.6.STABLE2/src/access_log.c Wed Jul 26 14:21:16 2006
+++ squid-2.6.STABLE3/src/access_log.c Wed Aug 2 19:26:18 2006
@@ -1,6 +1,6 @@
/*
- * $Id: access_log.c,v 1.85 2006/07/26 20:21:16 hno Exp $
+ * $Id: access_log.c,v 1.88 2006/08/03 01:26:18 hno Exp $
*
* DEBUG: section 46 Access Log
* AUTHOR: Duane Wessels
@@ -309,7 +309,9 @@
/*LFT_USER_SCHEME, */
LFT_USER_IDENT,
LFT_USER_EXT,
+#if USE_SSL
LFT_USER_SSL,
+#endif
LFT_HTTP_CODE,
/*LFT_HTTP_STATUS, */
@@ -407,6 +409,10 @@
/*{ "ur", LFT_USER_REALM }, */
/*{ "us", LFT_USER_SCHEME }, */
{"ui", LFT_USER_IDENT},
+#if USE_SSL
+ {"us", LFT_USER_SSL},
+#endif
+ {"ue", LFT_USER_EXT},
{"Hs", LFT_HTTP_CODE},
/*{ "Ht", LFT_HTTP_STATUS }, */
@@ -429,13 +435,13 @@
/*{ ">sb", LFT_REQUEST_SIZE_BODY }, */
/*{ ">sB", LFT_REQUEST_SIZE_BODY_NO_TE }, */
+ {"ea", LFT_EXT_LOG},
+
{"s_addr);
- if (!(a & 0xFFFFFFFFul))
- mask->s_addr = htonl(0x00000000ul);
- else if (!(a & 0x00FFFFFF))
- mask->s_addr = htonl(0xFF000000ul);
- else if (!(a & 0x0000FFFF))
- mask->s_addr = htonl(0xFFFF0000ul);
- else if (!(a & 0x000000FF))
- mask->s_addr = htonl(0xFFFFFF00ul);
- else
- mask->s_addr = htonl(0xFFFFFFFFul);
- }
return 1;
}
@@ -487,6 +471,7 @@
q->mask.s_addr = 0;
return q;
}
+ q->mask.s_addr = no_addr.s_addr; /* 255.255.255.255 */
if (sscanf(t, SCAN_ACL1, addr1, addr2, mask) == 3) {
(void) 0;
} else if (sscanf(t, SCAN_ACL2, addr1, addr2, &c) == 2) {
@@ -525,7 +510,7 @@
return NULL;
}
/* Decode addr1 */
- if (!decode_addr(addr1, &q->addr1, &q->mask)) {
+ if (!decode_addr(addr1, &q->addr1)) {
debug(28, 0) ("%s line %d: %s\n",
cfg_filename, config_lineno, config_input_line);
debug(28, 0) ("aclParseIpData: Ignoring invalid IP acl entry: unknown first address '%s'\n", addr1);
@@ -533,7 +518,7 @@
return NULL;
}
/* Decode addr2 */
- if (*addr2 && !decode_addr(addr2, &q->addr2, &q->mask)) {
+ if (*addr2 && !decode_addr(addr2, &q->addr2)) {
debug(28, 0) ("%s line %d: %s\n",
cfg_filename, config_lineno, config_input_line);
debug(28, 0) ("aclParseIpData: Ignoring invalid IP acl entry: unknown second address '%s'\n", addr2);
@@ -541,7 +526,7 @@
return NULL;
}
/* Decode mask */
- if (*mask && !decode_addr(mask, &q->mask, NULL)) {
+ if (*mask && !decode_addr(mask, &q->mask)) {
debug(28, 0) ("%s line %d: %s\n",
cfg_filename, config_lineno, config_input_line);
debug(28, 0) ("aclParseIpData: Ignoring invalid IP acl entry: unknown netmask '%s'\n", mask);
@@ -568,10 +553,12 @@
splayNode **Top = curlist;
acl_ip_data *q = NULL;
while ((t = strtokFile())) {
- q = aclParseIpData(t);
- while (q != NULL) {
+ acl_ip_data *next;
+ for (q = aclParseIpData(t); q != NULL; q = next) {
+ next = q->next;
*Top = splay_insert(q, *Top, aclIpNetworkCompare);
- q = q->next;
+ if (splayLastResult == 0)
+ xfree(q);
}
}
}
@@ -620,7 +607,8 @@
default:
debug(28, 0) ("%s line %d: %s\n",
cfg_filename, config_lineno, config_input_line);
- debug(28, 0) ("aclParseTimeSpec: Bad Day '%c'\n", *t);
+ debug(28, 0) ("aclParseTimeSpec: Bad Day '%c'\n", t[-1]);
+ self_destruct();
break;
}
}
@@ -813,7 +801,10 @@
} else {
if (data->flags.case_insensitive)
Tolower(t);
- Top = splay_insert(xstrdup(t), Top, (SPLAYCMP *) strcmp);
+ t = xstrdup(t);
+ Top = splay_insert(t, Top, (SPLAYCMP *) strcmp);
+ if (splayLastResult == 0)
+ xfree(t);
}
debug(28, 3) ("aclParseUserList: Case-insensitive-switch is %d\n",
data->flags.case_insensitive);
@@ -824,7 +815,10 @@
debug(28, 6) ("aclParseUserList: Got token: %s\n", t);
if (data->flags.case_insensitive)
Tolower(t);
- Top = splay_insert(xstrdup(t), Top, (SPLAYCMP *) strcmp);
+ t = xstrdup(t);
+ Top = splay_insert(t, Top, (SPLAYCMP *) strcmp);
+ if (splayLastResult == 0)
+ xfree(t);
}
data->names = Top;
}
@@ -841,7 +835,10 @@
splayNode **Top = curlist;
while ((t = strtokFile())) {
Tolower(t);
- *Top = splay_insert(xstrdup(t), *Top, aclDomainCompare);
+ t = xstrdup(t);
+ *Top = splay_insert(t, *Top, aclDomainCompare);
+ if (splayLastResult == 0)
+ safe_free(t);
}
}
@@ -864,7 +861,10 @@
}
Top = &(*datap)->values;
while ((t = strtokFile())) {
- *Top = splay_insert(xstrdup(t), *Top, aclDomainCompare);
+ t = xstrdup(t);
+ *Top = splay_insert(t, *Top, aclDomainCompare);
+ if (splayLastResult == 0)
+ safe_free(t);
}
}
@@ -3153,6 +3153,8 @@
if ((q = aclParseArpData(t)) == NULL)
continue;
*Top = splay_insert(q, *Top, aclArpCompare);
+ if (splayLastResult == 0)
+ safe_free(q);
}
}
diff -ruN squid-2.6.STABLE2/src/authenticate.c squid-2.6.STABLE3/src/authenticate.c
--- squid-2.6.STABLE2/src/authenticate.c Sun Jul 30 17:27:03 2006
+++ squid-2.6.STABLE3/src/authenticate.c Sat Aug 5 09:01:26 2006
@@ -1,6 +1,6 @@
/*
- * $Id: authenticate.c,v 1.48 2006/07/30 23:27:03 hno Exp $
+ * $Id: authenticate.c,v 1.49 2006/08/05 15:01:26 hno Exp $
*
* DEBUG: section 29 Authenticator
* AUTHOR: Duane Wessels
@@ -294,7 +294,7 @@
auth_user->ipcount++;
ip1 = xstrdup(inet_ntoa(ipaddr));
- debug(29, 2) ("authenticateAuthUserRequestSetIp: user '%s' has been seen at a new IP address (%s)\n ", authenticateUserUsername(auth_user), ip1);
+ debug(29, 2) ("authenticateAuthUserRequestSetIp: user '%s' has been seen at a new IP address (%s)\n", authenticateUserUsername(auth_user), ip1);
safe_free(ip1);
}
diff -ruN squid-2.6.STABLE2/src/cache_cf.c squid-2.6.STABLE3/src/cache_cf.c
--- squid-2.6.STABLE2/src/cache_cf.c Sun Jul 30 17:27:03 2006
+++ squid-2.6.STABLE3/src/cache_cf.c Sun Aug 6 13:00:19 2006
@@ -1,6 +1,6 @@
/*
- * $Id: cache_cf.c,v 1.452 2006/07/30 23:27:03 hno Exp $
+ * $Id: cache_cf.c,v 1.453 2006/08/06 19:00:19 hno Exp $
*
* DEBUG: section 3 Configuration File Parsing
* AUTHOR: Harvest Derived
@@ -667,21 +667,25 @@
static void
dump_acl(StoreEntry * entry, const char *name, acl * ae)
{
- wordlist *w;
- wordlist *v;
while (ae != NULL) {
debug(3, 3) ("dump_acl: %s %s\n", name, ae->name);
- v = w = aclDumpGeneric(ae);
- while (v != NULL) {
- debug(3, 3) ("dump_acl: %s %s %s\n", name, ae->name, v->key);
- storeAppendPrintf(entry, "%s %s %s %s\n",
- name,
- ae->name,
- aclTypeToStr(ae->type),
- v->key);
- v = v->next;
+ if (strstr(ae->cfgline, " \""))
+ storeAppendPrintf(entry, "%s\n", ae->cfgline);
+ else {
+ wordlist *w;
+ wordlist *v;
+ v = w = aclDumpGeneric(ae);
+ while (v != NULL) {
+ debug(3, 3) ("dump_acl: %s %s %s\n", name, ae->name, v->key);
+ storeAppendPrintf(entry, "%s %s %s %s\n",
+ name,
+ ae->name,
+ aclTypeToStr(ae->type),
+ v->key);
+ v = v->next;
+ }
+ wordlistDestroy(&w);
}
- wordlistDestroy(&w);
ae = ae->next;
}
}
diff -ruN squid-2.6.STABLE2/src/cf.data.pre squid-2.6.STABLE3/src/cf.data.pre
--- squid-2.6.STABLE2/src/cf.data.pre Sun Jul 30 19:31:18 2006
+++ squid-2.6.STABLE3/src/cf.data.pre Tue Aug 15 20:33:22 2006
@@ -1,6 +1,6 @@
#
-# $Id: cf.data.pre,v 1.362 2006/07/31 01:31:18 hno Exp $
+# $Id: cf.data.pre,v 1.364 2006/08/16 02:33:22 swilton Exp $
#
#
# SQUID Web Proxy Cache http://www.squid-cache.org/
@@ -1132,6 +1132,43 @@
leads to a maximum cache_dir size of 512<<24, or 8 GB. Note
you should not change the COSS block size after Squid
has written some objects to the cache_dir.
+
+ overwrite-percent=n defines the percentage of disk that COSS
+ must write to before a given object will be moved to the
+ current stripe. A value of "n" closer to 100 will cause COSS
+ to waste less disk space by having multiple copies of an object
+ on disk, but will increase the chances of overwriting a popular
+ object as COSS overwrites stripes. A value of "n" close to 0
+ will cause COSS to keep all current objects in the current COSS
+ stripe at the expense of the hit rate. The default value of 50
+ will allow any given object to be stored on disk a maximum of
+ 2 times.
+
+ max-stripe-waste=n defines the maximum amount of space that COSS
+ will waste in a given stripe (in bytes). When COSS writes data
+ to disk, it will potentially waste up to "max-size" worth of disk
+ space for each 1MB of data written. If "max-size" is set to a
+ large value (ie >256k), this could potentially result in large
+ amounts of wasted disk space. Setting this value to a lower value
+ (ie 64k or 32k) will result in a COSS disk refusing to cache
+ larger objects until the COSS stripe has been filled to within
+ "max-stripe-waste" of the maximum size (1MB).
+
+ membufs=n defines the number of "memory-only" stripes that COSS
+ will use. When an cache hit is performed on a COSS stripe before
+ COSS has reached the overwrite-percent value for that object,
+ COSS will use a series of memory buffers to hold the object in
+ while the data is sent to the client. This will define the maximum
+ number of memory-only buffers that COSS will use. The default value
+ is 10, which will use a maximum of 10MB of memory for buffers.
+
+ maxfullbufs=n defines the maximum number of stripes a COSS partition
+ will have in memory waiting to be freed (either because the disk is
+ under load and the stripe is unwritten, or because clients are still
+ transferring data from objects using the memory). In order to try
+ and maintain a good hit rate under load, COSS will reserve the last
+ 2 full stripes for object hits. (ie a COSS cache_dir will reject
+ new objects when the number of full stripes is 2 less than maxfullbufs)
Common options:
diff -ruN squid-2.6.STABLE2/src/client_side.c squid-2.6.STABLE3/src/client_side.c
--- squid-2.6.STABLE2/src/client_side.c Sun Jul 23 15:44:22 2006
+++ squid-2.6.STABLE3/src/client_side.c Tue Aug 15 18:40:20 2006
@@ -1,6 +1,6 @@
/*
- * $Id: client_side.c,v 1.666 2006/07/23 21:44:22 hno Exp $
+ * $Id: client_side.c,v 1.673 2006/08/16 00:40:20 hno Exp $
*
* DEBUG: section 33 Client-side Routines
* AUTHOR: Duane Wessels
@@ -2053,7 +2053,8 @@
return;
}
if (vary->key) {
- debug(33, 2) ("clientProcessVary: HIT key=%s etag=%s\n", vary->key, vary->etag);
+ debug(33, 2) ("clientProcessVary: HIT key=%s etag=%s\n",
+ vary->key, vary->etag ? vary->etag : "NONE");
} else {
int i;
debug(33, 2) ("clientProcessVary MISS\n");
@@ -3734,7 +3735,7 @@
http->flags.transparent = 1;
} else if (conn->port->accel) {
http->flags.accel = 1;
- if (!conn->port->vhost) {
+ if (!conn->port->vhost && strncasecmp(url, "cache_object://", 15) != 0) {
url = strstr(url, "//");
if (!url)
goto invalid_request;
@@ -3764,6 +3765,7 @@
/* This tries to back out what is done above */
dlinkDelete(&http->active, &ClientActiveRequests);
safe_free(http->uri);
+ xfree(inbuf);
cbdataFree(http);
return parseHttpRequestAbort(conn, "error:invalid-request");
}
@@ -4266,7 +4268,7 @@
ConnStateData *conn = data;
ErrorState *err;
debug(33, 3) ("requestTimeout: FD %d: lifetime is expired.\n", fd);
- if (fd_table[fd].rwstate) {
+ if (fd_table[fd].rwstate.valid) {
/*
* Some data has been sent to the client, just close the FD
*/
@@ -4325,7 +4327,7 @@
}
static int
-httpAcceptDefer(int fdunused, void *dataunused)
+httpAcceptDefer(int fd, void *dataunused)
{
static time_t last_warn = 0;
if (fdNFree() >= RESERVED_FD)
@@ -4334,6 +4336,7 @@
debug(33, 0) ("WARNING! Your cache is running out of filedescriptors\n");
last_warn = squid_curtime;
}
+ commDeferFD(fd);
return 1;
}
@@ -4628,7 +4631,7 @@
/* Write out the SSL session details.. actually the call below, but
* OpenSSL headers do strange typecasts confusing GCC.. */
/* PEM_write_SSL_SESSION(debug_log, SSL_get_session(ssl)); */
-#if defined(OPENSSL_VERSION_NUMBER) && OPENSSL_VERSION_NUMBER >= 0x0090708FL
+#if defined(OPENSSL_VERSION_NUMBER) && OPENSSL_VERSION_NUMBER >= 0x00908000L
PEM_ASN1_write((i2d_of_void *) i2d_SSL_SESSION, PEM_STRING_SSL_SESSION, debug_log, (char *) SSL_get_session(ssl), NULL, NULL, 0, NULL, NULL);
#else
PEM_ASN1_write(i2d_SSL_SESSION, PEM_STRING_SSL_SESSION, debug_log, (char *) SSL_get_session(ssl), NULL, NULL, 0, NULL, NULL);
@@ -5036,9 +5039,12 @@
else if (conn->pinning.fd != -1)
comm_close(conn->pinning.fd);
conn->pinning.fd = fd;
+ safe_free(conn->pinning.host);
conn->pinning.host = xstrdup(host);
conn->pinning.port = port;
conn->pinning.pinned = 1;
+ if (conn->pinning.peer)
+ cbdataUnlock(conn->pinning.peer);
conn->pinning.peer = peer;
if (peer)
cbdataLock(conn->pinning.peer);
diff -ruN squid-2.6.STABLE2/src/comm.c squid-2.6.STABLE3/src/comm.c
--- squid-2.6.STABLE2/src/comm.c Tue Jun 27 05:14:22 2006
+++ squid-2.6.STABLE3/src/comm.c Sat Aug 12 17:27:20 2006
@@ -1,6 +1,6 @@
/*
- * $Id: comm.c,v 1.353 2006/06/27 11:14:22 hno Exp $
+ * $Id: comm.c,v 1.355 2006/08/12 23:27:20 hno Exp $
*
* DEBUG: section 5 Socket Functions
* AUTHOR: Harvest Derived
@@ -79,12 +79,13 @@
static void
CommWriteStateCallbackAndFree(int fd, int code)
{
- CommWriteStateData *CommWriteState = fd_table[fd].rwstate;
+ CommWriteStateData *CommWriteState = &fd_table[fd].rwstate;
CWCB *callback = NULL;
void *data;
- fd_table[fd].rwstate = NULL;
- if (CommWriteState == NULL)
+ if (!CommWriteState->valid) {
return;
+ }
+ CommWriteState->valid = 0;
if (CommWriteState->free_func) {
FREE *free_func = CommWriteState->free_func;
void *free_buf = CommWriteState->buf;
@@ -95,10 +96,10 @@
callback = CommWriteState->handler;
data = CommWriteState->handler_data;
CommWriteState->handler = NULL;
+ CommWriteState->valid = 0;
if (callback && cbdataValid(data))
callback(fd, CommWriteState->buf, CommWriteState->offset, code, data);
cbdataUnlock(data);
- memPoolFree(comm_write_pool, CommWriteState);
}
/* Return the local port associated with fd. */
@@ -1041,9 +1042,11 @@
static void
commHandleWrite(int fd, void *data)
{
- CommWriteStateData *state = data;
int len = 0;
int nleft;
+ CommWriteStateData *state = &fd_table[fd].rwstate;
+
+ assert(state->valid);
debug(5, 5) ("commHandleWrite: FD %d: off %ld, sz %ld.\n",
fd, (long int) state->offset, (long int) state->size);
@@ -1072,7 +1075,7 @@
commSetSelect(fd,
COMM_SELECT_WRITE,
commHandleWrite,
- state,
+ NULL,
0);
} else {
debug(5, 2) ("commHandleWrite: FD %d: write failure: %s.\n",
@@ -1087,7 +1090,7 @@
commSetSelect(fd,
COMM_SELECT_WRITE,
commHandleWrite,
- state,
+ NULL,
0);
} else {
CommWriteStateCallbackAndFree(fd, COMM_OK);
@@ -1102,23 +1105,22 @@
void
comm_write(int fd, const char *buf, int size, CWCB * handler, void *handler_data, FREE * free_func)
{
- CommWriteStateData *state = fd_table[fd].rwstate;
+ CommWriteStateData *state = &fd_table[fd].rwstate;
debug(5, 5) ("comm_write: FD %d: sz %d: hndl %p: data %p.\n",
fd, size, handler, handler_data);
- if (NULL != state) {
- debug(5, 1) ("comm_write: fd_table[%d].rwstate != NULL\n", fd);
- memPoolFree(comm_write_pool, state);
- fd_table[fd].rwstate = NULL;
+ if (state->valid) {
+ debug(5, 1) ("comm_write: fd_table[%d].rwstate.valid == true!\n", fd);
+ fd_table[fd].rwstate.valid = 0;
}
- fd_table[fd].rwstate = state = memPoolAlloc(comm_write_pool);
state->buf = (char *) buf;
state->size = size;
state->offset = 0;
state->handler = handler;
state->handler_data = handler_data;
state->free_func = free_func;
+ state->valid = 1;
cbdataLock(handler_data);
- commSetSelect(fd, COMM_SELECT_WRITE, commHandleWrite, state, 0);
+ commSetSelect(fd, COMM_SELECT_WRITE, commHandleWrite, NULL, 0);
}
/* a wrapper around comm_write to allow for MemBuf to be comm_written in a snap */
diff -ruN squid-2.6.STABLE2/src/comm_epoll.c squid-2.6.STABLE3/src/comm_epoll.c
--- squid-2.6.STABLE2/src/comm_epoll.c Tue Jun 27 07:09:43 2006
+++ squid-2.6.STABLE3/src/comm_epoll.c Tue Aug 15 13:27:28 2006
@@ -1,6 +1,6 @@
/*
- * $Id: comm_epoll.c,v 1.24 2006/06/27 13:09:43 hno Exp $
+ * $Id: comm_epoll.c,v 1.26 2006/08/15 19:27:28 hno Exp $
*
* DEBUG: section 5 Socket Functions
*
@@ -78,12 +78,24 @@
}
void
+comm_select_postinit()
+{
+ debug(5, 1) ("Using epoll for the IO loop\n");
+}
+
+void
comm_select_shutdown()
{
fd_close(kdpfd);
close(kdpfd);
kdpfd = -1;
safe_free(epoll_state);
+}
+
+void
+comm_select_status(StoreEntry * sentry)
+{
+ storeAppendPrintf(sentry, "\tIO loop method: epoll\n");
}
void
diff -ruN squid-2.6.STABLE2/src/comm_kqueue.c squid-2.6.STABLE3/src/comm_kqueue.c
--- squid-2.6.STABLE2/src/comm_kqueue.c Tue Jun 27 07:09:43 2006
+++ squid-2.6.STABLE3/src/comm_kqueue.c Tue Aug 15 13:27:28 2006
@@ -1,6 +1,6 @@
/*
- * $Id: comm_kqueue.c,v 1.7 2006/06/27 13:09:43 hno Exp $
+ * $Id: comm_kqueue.c,v 1.9 2006/08/15 19:27:28 hno Exp $
*
* DEBUG: section 5 Socket Functions
*
@@ -66,12 +66,24 @@
}
void
+comm_select_postinit()
+{
+ debug(5, 1) ("Using kqueue for the IO loop\n");
+}
+
+void
comm_select_shutdown()
{
fd_close(kq);
close(kq);
kq = -1;
safe_free(kqueue_state);
+}
+
+void
+comm_select_status(StoreEntry * sentry)
+{
+ storeAppendPrintf(sentry, "\tIO loop method: kqueue\n");
}
void
diff -ruN squid-2.6.STABLE2/src/comm_poll.c squid-2.6.STABLE3/src/comm_poll.c
--- squid-2.6.STABLE2/src/comm_poll.c Wed Jul 19 09:56:39 2006
+++ squid-2.6.STABLE3/src/comm_poll.c Tue Aug 15 13:27:28 2006
@@ -1,6 +1,6 @@
/*
- * $Id: comm_poll.c,v 1.19 2006/07/19 15:56:39 hno Exp $
+ * $Id: comm_poll.c,v 1.21 2006/08/15 19:27:28 hno Exp $
*
* DEBUG: section 5 Socket Functions
*
@@ -576,8 +576,20 @@
}
void
+comm_select_postinit()
+{
+ debug(5, 1) ("Using poll for the IO loop\n");
+}
+
+void
comm_select_shutdown(void)
{
+}
+
+void
+comm_select_status(StoreEntry * sentry)
+{
+ storeAppendPrintf(sentry, "\tIO loop method: poll\n");
}
static void
diff -ruN squid-2.6.STABLE2/src/comm_select.c squid-2.6.STABLE3/src/comm_select.c
--- squid-2.6.STABLE2/src/comm_select.c Thu Jun 8 06:53:20 2006
+++ squid-2.6.STABLE3/src/comm_select.c Tue Aug 15 13:27:28 2006
@@ -1,6 +1,6 @@
/*
- * $Id: comm_select.c,v 1.75 2006/06/08 12:53:20 hno Exp $
+ * $Id: comm_select.c,v 1.77 2006/08/15 19:27:28 hno Exp $
*
* DEBUG: section 5 Socket Functions
*
@@ -569,8 +569,20 @@
}
void
+comm_select_postinit()
+{
+ debug(5, 1) ("Using select for the IO loop\n");
+}
+
+void
comm_select_shutdown(void)
{
+}
+
+void
+comm_select_status(StoreEntry * sentry)
+{
+ storeAppendPrintf(sentry, "\tIO loop method: select\n");
}
/*
diff -ruN squid-2.6.STABLE2/src/comm_select_win32.c squid-2.6.STABLE3/src/comm_select_win32.c
--- squid-2.6.STABLE2/src/comm_select_win32.c Sun Jun 11 11:06:25 2006
+++ squid-2.6.STABLE3/src/comm_select_win32.c Tue Aug 15 14:28:20 2006
@@ -1,6 +1,6 @@
/*
- * $Id: comm_select_win32.c,v 1.1 2006/06/11 17:06:25 serassio Exp $
+ * $Id: comm_select_win32.c,v 1.2 2006/08/15 20:28:20 serassio Exp $
*
* DEBUG: section 5 Socket Functions
*
@@ -607,8 +607,20 @@
}
void
+comm_select_postinit()
+{
+ debug(5, 1) ("Using select for the IO loop\n");
+}
+
+void
comm_select_shutdown(void)
{
+}
+
+void
+comm_select_status(StoreEntry * sentry)
+{
+ storeAppendPrintf(sentry, "\tIO loop method: select\n");
}
/*
diff -ruN squid-2.6.STABLE2/src/defines.h squid-2.6.STABLE3/src/defines.h
--- squid-2.6.STABLE2/src/defines.h Wed Jun 28 04:31:56 2006
+++ squid-2.6.STABLE3/src/defines.h Wed Aug 2 20:31:11 2006
@@ -1,6 +1,6 @@
/*
- * $Id: defines.h,v 1.118 2006/06/28 10:31:56 hno Exp $
+ * $Id: defines.h,v 1.120 2006/08/03 02:31:11 adrian Exp $
*
*
* SQUID Web Proxy Cache http://www.squid-cache.org/
@@ -41,6 +41,25 @@
#define FALSE 0
#endif
+/* Define load weights for cache_dir types */
+#define MAX_LOAD_VALUE 1000
+
+#define COSS_LOAD_BASE 0
+#define AUFS_LOAD_BASE 100
+#define DISKD_LOAD_BASE 100
+#define UFS_LOAD_BASE 500
+
+#define COSS_LOAD_STRIPE_WEIGHT (900 - COSS_LOAD_BASE)
+#define COSS_LOAD_QUEUE_WEIGHT (100 - COSS_LOAD_BASE)
+#if COSS_LOAD_QUEUE_WEIGHT < 0
+#undef COSS_LOAD_QUEUE_WEIGHT
+#define COSS_LOAD_QUEUE_WEIGHT 0
+#endif
+
+#define AUFS_LOAD_QUEUE_WEIGHT (MAX_LOAD_VALUE - AUFS_LOAD_BASE)
+
+#define DISKD_LOAD_QUEUE_WEIGHT (MAX_LOAD_VALUE - DISKD_LOAD_BASE)
+
#define ACL_NAME_SZ 32
#define BROWSERNAMELEN 128
@@ -323,5 +342,9 @@
#else
#define FILE_MODE(x) ((x)&(O_RDONLY|O_WRONLY|O_RDWR))
#endif
+
+/* swap_filen is 25 bits, signed */
+#define FILEMAP_MAX_SIZE (1<<24)
+#define FILEMAP_MAX (FILEMAP_MAX_SIZE - 65536)
#endif /* SQUID_DEFINES_H */
diff -ruN squid-2.6.STABLE2/src/delay_pools.c squid-2.6.STABLE3/src/delay_pools.c
--- squid-2.6.STABLE2/src/delay_pools.c Thu Jun 8 13:36:36 2006
+++ squid-2.6.STABLE3/src/delay_pools.c Wed Aug 2 15:33:51 2006
@@ -1,6 +1,6 @@
/*
- * $Id: delay_pools.c,v 1.32 2006/06/08 19:36:36 hno Exp $
+ * $Id: delay_pools.c,v 1.33 2006/08/02 21:33:51 hno Exp $
*
* DEBUG: section 77 Delay Pools
* AUTHOR: David Luyer
@@ -657,8 +657,6 @@
dlink_node *node;
for (node = mem->clients.head; node; node = node->next) {
sc = (store_client *) node->data;
- if (sc->callback_data == NULL) /* open slot */
- continue;
i = delayBytesWanted(sc->delay_id, i, max);
found = 1;
}
@@ -675,8 +673,6 @@
delay_id d = 0;
for (node = mem->clients.head; node; node = node->next) {
sc = (store_client *) node->data;
- if (sc->callback_data == NULL) /* open slot */
- continue;
j = delayBytesWanted(sc->delay_id, 0, INT_MAX);
if (j > jmax) {
jmax = j;
diff -ruN squid-2.6.STABLE2/src/dns_internal.c squid-2.6.STABLE3/src/dns_internal.c
--- squid-2.6.STABLE2/src/dns_internal.c Mon Jun 26 09:01:59 2006
+++ squid-2.6.STABLE3/src/dns_internal.c Tue Aug 15 18:27:20 2006
@@ -1,6 +1,6 @@
/*
- * $Id: dns_internal.c,v 1.58 2006/06/26 15:01:59 hno Exp $
+ * $Id: dns_internal.c,v 1.59 2006/08/16 00:27:20 hno Exp $
*
* DEBUG: section 78 DNS lookups; interacts with lib/rfc1035.c
* AUTHOR: Duane Wessels
@@ -83,6 +83,7 @@
static int RcodeMatrix[MAX_RCODE][MAX_ATTEMPT];
typedef struct _idns_query idns_query;
+CBDATA_TYPE(idns_query);
typedef struct _ns ns;
typedef struct _sp sp;
@@ -107,13 +108,16 @@
idns_query *queue;
unsigned short domain;
unsigned short do_searchpath;
+ int tcp_socket;
+ char *tcp_buffer;
+ size_t tcp_buffer_size;
+ size_t tcp_buffer_offset;
};
struct _ns {
struct sockaddr_in S;
int nqueries;
int nreplies;
- int large_pkts;
};
struct _sp {
@@ -531,6 +535,19 @@
}
static void
+idnsTcpCleanup(idns_query * q)
+{
+ if (q->tcp_socket != -1) {
+ comm_close(q->tcp_socket);
+ q->tcp_socket = -1;
+ }
+ if (q->tcp_buffer) {
+ memFreeBuf(q->tcp_buffer_size, q->tcp_buffer);
+ q->tcp_buffer = NULL;
+ }
+}
+
+static void
idnsSendQuery(idns_query * q)
{
int x;
@@ -543,6 +560,7 @@
assert(nns > 0);
assert(q->lru.next == NULL);
assert(q->lru.prev == NULL);
+ idnsTcpCleanup(q);
try_again:
ns = q->nsends % nns;
x = comm_udp_sendto(DnsSocket,
@@ -627,7 +645,7 @@
cbdataUnlock(q2->callback_data);
if (valid)
q2->callback(q2->callback_data, answers, n, error);
- memFree(q2, MEM_IDNS_QUERY);
+ cbdataFree(q2);
}
if (q->hash.key) {
hash_remove_link(idns_lookup_hash, &q->hash);
@@ -636,6 +654,100 @@
}
static void
+idnsReadTcp(int fd, void *data)
+{
+ ssize_t n;
+ idns_query *q = data;
+ int ns = (q->nsends - 1) % nns;
+ if (!q->tcp_buffer)
+ q->tcp_buffer = memAllocBuf(1024, &q->tcp_buffer_size);
+ statCounter.syscalls.sock.reads++;
+ n = FD_READ_METHOD(q->tcp_socket, q->tcp_buffer + q->tcp_buffer_offset, q->tcp_buffer_size - q->tcp_buffer_offset);
+ if (n < 0 && ignoreErrno(errno)) {
+ commSetSelect(q->tcp_socket, COMM_SELECT_READ, idnsReadTcp, q, 0);
+ return;
+ }
+ if (n <= 0) {
+ debug(78, 2) ("idnsReadTcp: Short response for %s.\n", q->name);
+ dlinkDelete(&q->lru, &lru_list);
+ idnsSendQuery(q);
+ return;
+ }
+ fd_bytes(fd, n, FD_READ);
+ q->tcp_buffer_offset += n;
+ if (q->tcp_buffer_offset > 2) {
+ unsigned short response_size = ntohs(*(short *) q->tcp_buffer);
+ if (q->tcp_buffer_offset >= response_size + 2) {
+ nameservers[ns].nreplies++;
+ idnsGrokReply(q->tcp_buffer + 2, response_size);
+ return;
+ }
+ if (q->tcp_buffer_size < response_size + 2)
+ q->tcp_buffer = memReallocBuf(q->tcp_buffer, response_size + 2, &q->tcp_buffer_size);
+ }
+ commSetSelect(q->tcp_socket, COMM_SELECT_READ, idnsReadTcp, q, 0);
+}
+
+static void
+idnsSendTcpQueryDone(int fd, char *bufnotused, size_t size, int errflag, void *data)
+{
+ idns_query *q = data;
+ if (size > 0)
+ fd_bytes(fd, size, FD_WRITE);
+ if (errflag == COMM_ERR_CLOSING)
+ return;
+ if (errflag) {
+ dlinkDelete(&q->lru, &lru_list);
+ idnsSendQuery(q);
+ return;
+ }
+ commSetSelect(q->tcp_socket, COMM_SELECT_READ, idnsReadTcp, q, 0);
+}
+
+static void
+idnsSendTcpQuery(int fd, int status, void *data)
+{
+ MemBuf buf;
+ idns_query *q = data;
+ short nsz;
+ if (status != COMM_OK) {
+ dlinkDelete(&q->lru, &lru_list);
+ idnsSendQuery(q);
+ return;
+ }
+ memBufInit(&buf, q->sz + 2, q->sz + 2);
+ nsz = htons(q->sz);
+ memBufAppend(&buf, &nsz, 2);
+ memBufAppend(&buf, q->buf, q->sz);
+ comm_write_mbuf(q->tcp_socket, buf, idnsSendTcpQueryDone, q);
+}
+
+static void
+idnsRetryTcp(idns_query * q)
+{
+ struct in_addr addr;
+ int ns = (q->nsends - 1) % nns;
+ idnsTcpCleanup(q);
+ if (Config.Addrs.udp_outgoing.s_addr != no_addr.s_addr)
+ addr = Config.Addrs.udp_outgoing;
+ else
+ addr = Config.Addrs.udp_incoming;
+ q->tcp_socket = comm_open(SOCK_STREAM,
+ IPPROTO_TCP,
+ addr,
+ 0,
+ COMM_NONBLOCKING,
+ "DNS TCP Socket");
+ dlinkAdd(q, &q->lru, &lru_list);
+ commConnectStart(q->tcp_socket,
+ inet_ntoa(nameservers[ns].S.sin_addr),
+ ntohs(nameservers[ns].S.sin_port),
+ idnsSendTcpQuery,
+ q
+ );
+}
+
+static void
idnsGrokReply(const char *buf, size_t sz)
{
int n;
@@ -663,6 +775,12 @@
return;
}
dlinkDelete(&q->lru, &lru_list);
+ if (message->tc && q->tcp_socket == -1) {
+ debug(78, 2) ("idnsGrokReply: Response for %s truncated. Retrying using TCP\n", message->query->name);
+ rfc1035MessageDestroy(message);
+ idnsRetryTcp(q);
+ return;
+ }
idnsRcodeCount(n, q->attempt);
q->error = NULL;
if (n < 0) {
@@ -709,7 +827,8 @@
idnsCallback(q, message->answer, n, q->error);
rfc1035MessageDestroy(message);
- memFree(q, MEM_IDNS_QUERY);
+ idnsTcpCleanup(q);
+ cbdataFree(q);
}
static void
@@ -796,7 +915,8 @@
idnsCallback(q, NULL, -q->rcode, q->error);
else
idnsCallback(q, NULL, -16, "Timeout");
- memFree(q, MEM_IDNS_QUERY);
+ idnsTcpCleanup(q);
+ cbdataFree(q);
}
}
idnsTickleQueue();
@@ -823,6 +943,7 @@
idnsInit(void)
{
static int init = 0;
+ CBDATA_INIT_TYPE(idns_query);
if (DnsSocket < 0) {
int port;
struct in_addr addr;
@@ -895,7 +1016,8 @@
idns_query *old = hash_lookup(idns_lookup_hash, key);
if (!old)
return 0;
- q = memAllocate(MEM_IDNS_QUERY);
+ q = cbdataAlloc(idns_query);
+ q->tcp_socket = -1;
q->callback = callback;
q->callback_data = data;
cbdataLock(q->callback_data);
@@ -919,7 +1041,8 @@
idns_query *q;
if (idnsCachedLookup(name, callback, data))
return;
- q = memAllocate(MEM_IDNS_QUERY);
+ q = cbdataAlloc(idns_query);
+ q->tcp_socket = -1;
q->id = idnsQueryID();
for (i = 0; i < strlen(name); i++) {
@@ -948,7 +1071,7 @@
if (q->sz < 0) {
/* problem with query data -- query not sent */
callback(data, NULL, 0, "Internal error");
- memFree(q, MEM_IDNS_QUERY);
+ cbdataFree(q);
return;
}
debug(78, 3) ("idnsALookup: buf is %d bytes for %s, id = %#hx\n",
@@ -968,7 +1091,8 @@
const char *ip = inet_ntoa(addr);
if (idnsCachedLookup(ip, callback, data))
return;
- q = memAllocate(MEM_IDNS_QUERY);
+ q = cbdataAlloc(idns_query);
+ q->tcp_socket = -1;
q->id = idnsQueryID();
q->sz = rfc1035BuildPTRQuery(addr, q->buf, sizeof(q->buf), q->id, &q->query);
debug(78, 3) ("idnsPTRLookup: buf is %d bytes for %s, id = %#hx\n",
@@ -976,7 +1100,7 @@
if (q->sz < 0) {
/* problem with query data -- query not sent */
callback(data, NULL, 0, "Internal error");
- memFree(q, MEM_IDNS_QUERY);
+ cbdataFree(q);
return;
}
q->callback = callback;
diff -ruN squid-2.6.STABLE2/src/filemap.c squid-2.6.STABLE3/src/filemap.c
--- squid-2.6.STABLE2/src/filemap.c Wed Oct 24 01:45:34 2001
+++ squid-2.6.STABLE3/src/filemap.c Mon Jul 31 04:29:44 2006
@@ -1,6 +1,6 @@
/*
- * $Id: filemap.c,v 1.39 2001/10/24 07:45:34 hno Exp $
+ * $Id: filemap.c,v 1.40 2006/07/31 10:29:44 hno Exp $
*
* DEBUG: section 8 Swap File Bitmap
* AUTHOR: Harvest Derived
@@ -75,7 +75,7 @@
int old_sz = fm->nwords * sizeof(*fm->file_map);
void *old_map = fm->file_map;
fm->max_n_files <<= 1;
- assert(fm->max_n_files <= (1 << 24)); /* swap_filen is 25 bits, signed */
+ assert(fm->max_n_files <= FILEMAP_MAX_SIZE); /* swap_filen is 25 bits, signed */
fm->nwords = fm->max_n_files >> LONG_BIT_SHIFT;
debug(8, 3) ("file_map_grow: creating space for %d files\n", fm->max_n_files);
fm->file_map = xcalloc(fm->nwords, sizeof(*fm->file_map));
diff -ruN squid-2.6.STABLE2/src/fs/aufs/store_dir_aufs.c squid-2.6.STABLE3/src/fs/aufs/store_dir_aufs.c
--- squid-2.6.STABLE2/src/fs/aufs/store_dir_aufs.c Sun Jul 30 17:27:04 2006
+++ squid-2.6.STABLE3/src/fs/aufs/store_dir_aufs.c Wed Aug 2 20:31:12 2006
@@ -1,6 +1,6 @@
/*
- * $Id: store_dir_aufs.c,v 1.59 2006/07/30 23:27:04 hno Exp $
+ * $Id: store_dir_aufs.c,v 1.62 2006/08/03 02:31:12 adrian Exp $
*
* DEBUG: section 47 Store Directory Routines
* AUTHOR: Duane Wessels
@@ -1520,6 +1520,7 @@
void
storeAufsDirMaintain(SwapDir * SD)
{
+ squidaioinfo_t *aioinfo = (squidaioinfo_t *) SD->fsdata;
StoreEntry *e = NULL;
int removed = 0;
int max_scan;
@@ -1542,7 +1543,7 @@
f, max_scan, max_remove);
walker = SD->repl->PurgeInit(SD->repl, max_scan);
while (1) {
- if (SD->cur_size < SD->low_size)
+ if (SD->cur_size < SD->low_size && aioinfo->map->n_files_in_map < FILEMAP_MAX)
break;
if (removed >= max_remove)
break;
@@ -1577,9 +1578,9 @@
ql = aioQueueSize();
if (ql == 0) {
- return 1;
+ return AUFS_LOAD_BASE;
}
- loadav = ql * 1000 / MAGIC1;
+ loadav = AUFS_LOAD_BASE + (ql * AUFS_LOAD_QUEUE_WEIGHT / MAGIC1);
return loadav;
}
@@ -1680,6 +1681,7 @@
storeAppendPrintf(sentry, "Current Size: %d KB\n", SD->cur_size);
storeAppendPrintf(sentry, "Percent Used: %0.2f%%\n",
100.0 * SD->cur_size / SD->max_size);
+ storeAppendPrintf(sentry, "Current load metric: %d / %d\n", storeAufsDirCheckLoadAv(SD, ST_OP_CREATE), MAX_LOAD_VALUE);
storeAppendPrintf(sentry, "Filemap bits in use: %d of %d (%d%%)\n",
aioinfo->map->n_files_in_map, aioinfo->map->max_n_files,
percent(aioinfo->map->n_files_in_map, aioinfo->map->max_n_files));
diff -ruN squid-2.6.STABLE2/src/fs/coss/store_coss.h squid-2.6.STABLE3/src/fs/coss/store_coss.h
--- squid-2.6.STABLE2/src/fs/coss/store_coss.h Wed Jul 5 00:52:12 2006
+++ squid-2.6.STABLE3/src/fs/coss/store_coss.h Tue Aug 15 20:33:22 2006
@@ -46,6 +46,7 @@
struct {
int alloc;
int realloc;
+ int memalloc;
int collisions;
} alloc;
int disk_overflows;
@@ -73,6 +74,7 @@
unsigned int writing:1;
unsigned int written:1;
unsigned int dead:1;
+ unsigned int memonly:1;
} flags;
int numobjs;
};
@@ -114,6 +116,7 @@
sfileno original_filen, new_filen; /* in blocks, not in bytes */
dlink_list ops;
char *p;
+ struct _cossmembuf *locked_membuf;
};
@@ -126,6 +129,8 @@
int fd;
int swaplog_fd;
int numcollisions;
+ int loadcalc[2];
+ int load_interval;
dlink_list pending_relocs;
dlink_list pending_ops;
int pending_reloc_count;
@@ -137,7 +142,14 @@
unsigned int blksz_bits;
unsigned int blksz_mask; /* just 1<fsdata;
+ int objsize = objectLen(e) + e->mem_obj->swap_hdr_sz;
/* Check if the object is a special object, we can't cache these */
if (EBIT_TEST(e->flags, ENTRY_SPECIAL))
return 0;
if (cs->rebuild.rebuilding == 1)
return 0;
+ /* Check to see if the object is going to waste too much disk space */
+ if (objsize > cs->sizerange_max)
+ return 0;
+
return 1;
}
int
storeCossDirCheckLoadAv(SwapDir * SD, store_op_t op)
{
-#if !USE_AUFSOPS
CossInfo *cs = (CossInfo *) SD->fsdata;
-#else
+#if USE_AUFSOPS
+ float disk_size_weight, current_write_weight;
+ int cur_load_interval = (squid_curtime / cs->load_interval) % 2;
int ql = 0;
#endif
int loadav;
@@ -592,13 +612,37 @@
#if USE_AUFSOPS
ql = aioQueueSize();
if (ql == 0)
- loadav = 0;
+ loadav = COSS_LOAD_BASE;
else
- loadav = ql * 1000 / MAGIC1;
+ loadav = COSS_LOAD_BASE + (ql * COSS_LOAD_QUEUE_WEIGHT / MAGIC1);
+
+ /* We want to try an keep the disks at a similar write rate
+ * otherwise the LRU algorithm breaks
+ *
+ * The queue length has a 10% weight on the load
+ * The number of stripes written has a 90% weight
+ */
+ disk_size_weight = (float) max_coss_dir_size / SD->max_size;
+ current_write_weight = (float) cs->loadcalc[cur_load_interval] * COSS_LOAD_STRIPE_WEIGHT / MAX_LOAD_VALUE;
+
+ loadav += disk_size_weight * current_write_weight;
+
+ /* Remove the folowing check if we want to allow COSS partitions to get
+ * too busy to accept new objects
+ */
+ if (loadav > MAX_LOAD_VALUE)
+ loadav = MAX_LOAD_VALUE;
+
+ /* Finally, we want to reject all new obects if the number of full stripes
+ * is too large
+ */
+ if (cs->numfullstripes > cs->hitonlyfullstripes)
+ loadav += MAX_LOAD_VALUE;
+
debug(47, 9) ("storeAufsDirCheckObj: load=%d\n", loadav);
return loadav;
#else
- loadav = cs->aq.aq_numpending * 1000 / MAX_ASYNCOP;
+ loadav = cs->aq.aq_numpending * MAX_LOAD_VALUE / MAX_ASYNCOP;
return loadav;
#endif
}
@@ -632,6 +676,7 @@
storeAppendPrintf(sentry, "Current Size: %d KB\n", SD->cur_size);
storeAppendPrintf(sentry, "Percent Used: %0.2f%%\n",
100.0 * SD->cur_size / SD->max_size);
+ storeAppendPrintf(sentry, "Current load metric: %d / %d\n", storeCossDirCheckLoadAv(SD, ST_OP_CREATE), MAX_LOAD_VALUE);
storeAppendPrintf(sentry, "Number of object collisions: %d\n", (int) cs->numcollisions);
#if 0
/* is this applicable? I Hope not .. */
@@ -716,7 +761,21 @@
cs->blksz_bits = 9; /* default block size = 512 */
cs->blksz_mask = (1 << cs->blksz_bits) - 1;
+ /* By default, only overwrite objects that were written mor ethan 50% of the disk ago
+ * and use a maximum of 10 in-memory stripes
+ */
+ cs->minumum_overwrite_pct = 0.5;
+ cs->nummemstripes = 10;
+
+ /* Calculate load in 60 second incremenets */
+ /* This could be made configurable */
+ cs->load_interval = 60;
+
parse_cachedir_options(sd, options, 0);
+
+ cs->sizerange_max = sd->max_objsize;
+ cs->sizerange_min = sd->max_objsize;
+
/* Enforce maxobjsize being set to something */
if (sd->max_objsize == -1)
fatal("COSS requires max-size to be set to something other than -1!\n");
@@ -728,12 +787,15 @@
* signed integer, as defined in structs.h.
*/
max_offset = (off_t) 0xFFFFFF << cs->blksz_bits;
- if (sd->max_size > (unsigned long) (max_offset >> 10)) {
+ if ((sd->max_size + cs->nummemstripes) > (unsigned long) (max_offset >> 10)) {
debug(47, 0) ("COSS block-size = %d bytes\n", 1 << cs->blksz_bits);
debug(47, 0) ("COSS largest file offset = %lu KB\n", (unsigned long) max_offset >> 10);
debug(47, 0) ("COSS cache_dir size = %d KB\n", sd->max_size);
fatal("COSS cache_dir size exceeds largest offset\n");
}
+ cs->max_disk_nf = ((off_t) sd->max_size << 10) >> cs->blksz_bits;
+ debug(47, 0) ("COSS: max disk fileno is %d\n", cs->max_disk_nf);
+
/* XXX todo checks */
/* Ensure that off_t range can cover the max_size */
@@ -748,6 +810,28 @@
cs->stripes[i].membuf = NULL;
cs->stripes[i].numdiskobjs = -1;
}
+ cs->minimum_stripe_distance = cs->numstripes * cs->minumum_overwrite_pct;
+
+ /* Make sure cs->maxfull has a default value */
+ if (cs->maxfullstripes == 0)
+ cs->maxfullstripes = cs->numstripes;
+
+ /* We will reject new objects (ie go into hit-only mode)
+ * if there are <= 2 stripes available
+ */
+ cs->hitonlyfullstripes = cs->maxfullstripes - HITONLY_BUFS;
+
+ debug(47, 0) ("COSS: number of memory-only stripes %d of %d bytes each\n", cs->nummemstripes, COSS_MEMBUF_SZ);
+ cs->memstripes = xcalloc(cs->nummemstripes, sizeof(struct _cossstripe));
+ for (i = 0; i < cs->nummemstripes; i++) {
+ cs->memstripes[i].id = i;
+ cs->memstripes[i].membuf = NULL;
+ cs->memstripes[i].numdiskobjs = -1;
+ }
+
+ /* Update the max size (used for load calculations) */
+ if (sd->max_size > max_coss_dir_size)
+ max_coss_dir_size = sd->max_size;
}
static void
@@ -776,9 +860,63 @@
void
storeCossDirDump(StoreEntry * entry, SwapDir * s)
{
- storeAppendPrintf(entry, " %d",
- s->max_size >> 20);
- dump_cachedir_options(entry, NULL, s);
+ storeAppendPrintf(entry, " %d", s->max_size >> 10);
+ dump_cachedir_options(entry, options, s);
+}
+
+static void
+storeCossDirParseMaxFullBufs(SwapDir * sd, const char *name, const char *value, int reconfiguring)
+{
+ CossInfo *cs = sd->fsdata;
+ int maxfull = atoi(value);
+ if (maxfull <= HITONLY_BUFS)
+ fatalf("COSS ERROR: There must be more than %d maxfullbufs\n", HITONLY_BUFS);
+ if (maxfull > 500)
+ fatal("COSS ERROR: Squid will likely use too much memory if it ever used 500MB worth of full buffers\n");
+ cs->maxfullstripes = maxfull;
+}
+
+static void
+storeCossDirParseMemOnlyBufs(SwapDir * sd, const char *name, const char *value, int reconfiguring)
+{
+ CossInfo *cs = sd->fsdata;
+ int membufs = atoi(value);
+ if (reconfiguring) {
+ debug(47, 0) ("WARNING: cannot change COSS memory bufs Squid is running\n");
+ return;
+ }
+ if (membufs < 2)
+ fatal("COSS ERROR: There must be at least 2 membufs\n");
+ if (membufs > 500)
+ fatal("COSS ERROR: Squid will likely use too much memory if it ever used 500MB worth of buffers\n");
+ cs->nummemstripes = membufs;
+}
+
+static void
+storeCossDirParseMaxWaste(SwapDir * sd, const char *name, const char *value, int reconfiguring)
+{
+ CossInfo *cs = sd->fsdata;
+ int waste = atoi(value);
+
+ if (waste < 8192)
+ fatal("COSS max-stripe-waste must be > 8192\n");
+ if (waste > sd->max_objsize)
+ debug(47, 1) ("storeCossDirParseMaxWaste: COSS max-stripe-waste can not be bigger than the max object size (%" PRINTF_OFF_T ")\n", sd->max_objsize);
+ cs->sizerange_min = waste;
+}
+
+static void
+storeCossDirParseOverwritePct(SwapDir * sd, const char *name, const char *value, int reconfiguring)
+{
+ CossInfo *cs = sd->fsdata;
+ int pct = atoi(value);
+
+ if (pct < 0)
+ fatal("COSS overwrite percent must be > 0\n");
+ if (pct > 100)
+ fatal("COSS overwrite percent must be < 100\n");
+ cs->minumum_overwrite_pct = (float) pct / 100;
+ cs->minimum_stripe_distance = cs->numstripes * cs->minumum_overwrite_pct;
}
static void
@@ -811,6 +949,34 @@
}
static void
+storeCossDirDumpMaxFullBufs(StoreEntry * e, const char *option, SwapDir * sd)
+{
+ CossInfo *cs = sd->fsdata;
+ storeAppendPrintf(e, " maxfullbufs=%d MB", cs->maxfullstripes);
+}
+
+static void
+storeCossDirDumpMemOnlyBufs(StoreEntry * e, const char *option, SwapDir * sd)
+{
+ CossInfo *cs = sd->fsdata;
+ storeAppendPrintf(e, " membufs=%d MB", cs->nummemstripes);
+}
+
+static void
+storeCossDirDumpMaxWaste(StoreEntry * e, const char *option, SwapDir * sd)
+{
+ CossInfo *cs = sd->fsdata;
+ storeAppendPrintf(e, " max-stripe-waste=%d", cs->sizerange_min);
+}
+
+static void
+storeCossDirDumpOverwritePct(StoreEntry * e, const char *option, SwapDir * sd)
+{
+ CossInfo *cs = sd->fsdata;
+ storeAppendPrintf(e, " overwrite-percent=%d%%", (int) cs->minumum_overwrite_pct * 100);
+}
+
+static void
storeCossDirDumpBlkSize(StoreEntry * e, const char *option, SwapDir * sd)
{
CossInfo *cs = sd->fsdata;
@@ -892,6 +1058,7 @@
storeAppendPrintf(sentry, "dead_stripes: %d\n", coss_stats.dead_stripes);
storeAppendPrintf(sentry, "alloc.alloc: %d\n", coss_stats.alloc.alloc);
storeAppendPrintf(sentry, "alloc.realloc: %d\n", coss_stats.alloc.realloc);
+ storeAppendPrintf(sentry, "alloc.memalloc: %d\n", coss_stats.alloc.memalloc);
storeAppendPrintf(sentry, "alloc.collisions: %d\n", coss_stats.alloc.collisions);
storeAppendPrintf(sentry, "disk_overflows: %d\n", coss_stats.disk_overflows);
storeAppendPrintf(sentry, "stripe_overflows: %d\n", coss_stats.stripe_overflows);
@@ -1126,9 +1293,9 @@
tmpe.hash.key = key;
/* Check sizes */
if (tmpe.swap_file_sz == 0) {
- tmpe.swap_file_sz = len;
+ tmpe.swap_file_sz = len + bl;
}
- if (tmpe.swap_file_sz != len) {
+ if (tmpe.swap_file_sz != (len + bl)) {
debug(47, 3) ("COSS: %s: stripe %d: file size mismatch (%" PRINTF_OFF_T " != %" PRINTF_OFF_T ")\n", SD->path, cs->rebuild.curstripe, tmpe.swap_file_sz, len);
goto nextobject;
}
diff -ruN squid-2.6.STABLE2/src/fs/coss/store_io_coss.c squid-2.6.STABLE3/src/fs/coss/store_io_coss.c
--- squid-2.6.STABLE2/src/fs/coss/store_io_coss.c Sun Jul 16 19:33:38 2006
+++ squid-2.6.STABLE3/src/fs/coss/store_io_coss.c Tue Aug 15 20:33:23 2006
@@ -1,6 +1,6 @@
/*
- * $Id: store_io_coss.c,v 1.25 2006/07/17 01:33:38 hno Exp $
+ * $Id: store_io_coss.c,v 1.28 2006/08/16 02:33:23 swilton Exp $
*
* DEBUG: section 79 Storage Manager COSS Interface
* AUTHOR: Eric Stern
@@ -52,6 +52,7 @@
static void storeCossMemBufUnlock(SwapDir * SD, storeIOState * e);
static void storeCossWriteMemBuf(SwapDir * SD, CossMemBuf * t);
static CossMemBuf *storeCossCreateMemBuf(SwapDir * SD, int stripe, sfileno curfn, int *collision);
+static CossMemBuf *storeCossCreateMemOnlyBuf(SwapDir * SD);
static CBDUNL storeCossIOFreeEntry;
static off_t storeCossFilenoToDiskOffset(sfileno f, CossInfo *);
static sfileno storeCossDiskOffsetToFileno(off_t o, CossInfo *);
@@ -81,6 +82,51 @@
/* === PUBLIC =========================================================== */
+static sfileno
+storeCossMemOnlyAllocate(SwapDir * SD, const StoreEntry * e)
+{
+ CossInfo *cs = (CossInfo *) SD->fsdata;
+ CossMemBuf *newmb;
+ off_t retofs;
+ size_t allocsize;
+ sfileno f;
+
+ coss_stats.alloc.memalloc++;
+ allocsize = e->swap_file_sz;
+
+ if (cs->current_memonly_membuf == NULL) {
+ newmb = storeCossCreateMemOnlyBuf(SD);
+ cs->current_memonly_membuf = newmb;
+
+ if (newmb == NULL) {
+ return -1;
+ }
+ cs->current_memonly_offset = cs->current_memonly_membuf->diskstart;
+ } else if ((cs->current_memonly_offset + allocsize) >= cs->current_memonly_membuf->diskend) {
+ debug(79, 3) ("storeCossMemOnlyAllocate: overflow for buffer %d (%p)\n", cs->curmemstripe, cs->current_memonly_membuf);
+ cs->current_memonly_membuf->flags.full = 1;
+ storeCossMaybeWriteMemBuf(SD, cs->current_memonly_membuf);
+ /* cs->current_memonly_membuf may be invalid at this point */
+
+ newmb = storeCossCreateMemOnlyBuf(SD);
+ cs->current_memonly_membuf = newmb;
+
+ if (newmb == NULL) {
+ return -1;
+ }
+ cs->current_memonly_offset = cs->current_memonly_membuf->diskstart;
+ }
+ retofs = cs->current_memonly_offset;
+ cs->current_memonly_offset = retofs + allocsize;
+ cs->current_memonly_membuf->numobjs++;
+ cs->current_memonly_offset = ((cs->current_memonly_offset + cs->blksz_mask) >> cs->blksz_bits) << cs->blksz_bits;
+ f = storeCossDiskOffsetToFileno(retofs, cs);
+ assert(f >= 0 && f <= 0xffffff);
+ debug(79, 3) ("storeCossMemOnlyAllocate: offset %lld, filen: %d\n", (long long int) retofs, f);
+ return f;
+
+}
+
/*
* This routine sucks. I want to rewrite it when possible, and I also think
* that we should check after creatmembuf() to see if the object has a
@@ -116,14 +162,29 @@
/* Since we're not supporting NOTIFY anymore, lets fail */
assert(which != COSS_ALLOC_NOTIFY);
- /* Check if we have overflowed the disk .. */
- if ((cs->current_offset + allocsize) > ((off_t) SD->max_size << 10)) {
+ /* Check to see if we need to allocate a membuf to start */
+ if (cs->current_membuf == NULL) {
+ if (cs->curstripe < cs->numstripes)
+ newmb = storeCossCreateMemBuf(SD, cs->curstripe + 1, checkf, &coll);
+ else
+ newmb = storeCossCreateMemBuf(SD, 0, checkf, &coll);
+
+ cs->current_membuf = newmb;
+ if (newmb == NULL) {
+ cs->sizerange_max = SD->max_objsize;
+ return -1;
+ }
+ cs->current_offset = cs->current_membuf->diskstart;
+
+ /* Check if we have overflowed the disk .. */
+ } else if ((cs->current_offset + allocsize) > ((off_t) SD->max_size << 10)) {
/*
* tried to allocate past the end of the disk, so wrap
* back to the beginning
*/
coss_stats.disk_overflows++;
cs->current_membuf->flags.full = 1;
+ cs->numfullstripes++;
cs->current_membuf->diskend = cs->current_offset;
storeCossMaybeWriteMemBuf(SD, cs->current_membuf);
/* cs->current_membuf may be invalid at this point */
@@ -132,7 +193,10 @@
newmb = storeCossCreateMemBuf(SD, 0, checkf, &coll);
cs->current_membuf = newmb;
-
+ if (newmb == NULL) {
+ cs->sizerange_max = SD->max_objsize;
+ return -1;
+ }
/* Check if we have overflowed the MemBuf */
} else if ((cs->current_offset + allocsize) >= cs->current_membuf->diskend) {
/*
@@ -140,6 +204,7 @@
*/
coss_stats.stripe_overflows++;
cs->current_membuf->flags.full = 1;
+ cs->numfullstripes++;
cs->current_offset = cs->current_membuf->diskend;
storeCossMaybeWriteMemBuf(SD, cs->current_membuf);
/* cs->current_membuf may be invalid at this point */
@@ -148,6 +213,10 @@
assert(cs->curstripe < (cs->numstripes - 1));
newmb = storeCossCreateMemBuf(SD, cs->curstripe + 1, checkf, &coll);
cs->current_membuf = newmb;
+ if (newmb == NULL) {
+ cs->sizerange_max = SD->max_objsize;
+ return -1;
+ }
}
/* If we didn't get a collision, then update the current offset and return it */
if (coll == 0) {
@@ -159,8 +228,20 @@
f = storeCossDiskOffsetToFileno(retofs, cs);
assert(f >= 0 && f <= 0xffffff);
debug(79, 3) ("storeCossAllocate: offset %lld, filen: %d\n", (long long int) retofs, f);
+
+ /*
+ * Keep track of the largest object we can accept based on the
+ * max-wasted-space value
+ */
+ cs->sizerange_max = cs->current_membuf->diskend - cs->current_offset;
+ if (cs->sizerange_max < cs->sizerange_min)
+ cs->sizerange_max = cs->sizerange_min;
+
return f;
} else {
+ /* Reset this to a safe value */
+ cs->sizerange_max = SD->max_objsize;
+
coss_stats.alloc.collisions++;
debug(79, 3) ("storeCossAllocate: %s: Collision\n", SD->path);
return -1;
@@ -183,7 +264,20 @@
storeCossUnlink(SD, e);
}
+static int
+storeCossRelocateRequired(CossInfo * cs, sfileno f)
+{
+ int stripes_written;
+ int original_stripe = storeCossFilenoToStripe(cs, f);
+
+ if (cs->curstripe > original_stripe)
+ stripes_written = cs->curstripe - original_stripe;
+ else
+ stripes_written = cs->numstripes + cs->curstripe - original_stripe;
+ /* Relocate if stripes_written > minimum_stripe_distance */
+ return (stripes_written > cs->minimum_stripe_distance);
+}
storeIOState *
storeCossCreate(SwapDir * SD, StoreEntry * e, STFNCB * file_callback, STIOCB * callback, void *callback_data)
@@ -280,7 +374,6 @@
storeCossMemBufLock(SD, sio);
debug(79, 3) ("storeCossOpen: %s: memory hit!\n", SD->path);
} else {
- debug(79, 3) ("storeCossOpen: %s: memory miss - doing reallocation\n", SD->path);
/* Do the allocation */
/* this is the first time we've been called on a new sio
* read the whole object into memory, then return the
@@ -294,7 +387,19 @@
*/
cstate->reqdiskoffset = storeCossFilenoToDiskOffset(sio->swap_filen, cs);
assert(cstate->reqdiskoffset >= 0);
- nf = storeCossAllocate(SD, e, COSS_ALLOC_REALLOC);
+
+ /* If the object is allocated too recently, make a memory-only copy */
+ if (storeCossRelocateRequired(cs, sio->swap_filen)) {
+ debug(79, 3) ("storeCossOpen: %s: memory miss - doing reallocation (Current stripe : %d Object in stripe : %d)\n", SD->path, cs->curstripe, storeCossFilenoToStripe(cs, sio->swap_filen));
+ nf = storeCossAllocate(SD, e, COSS_ALLOC_REALLOC);
+ } else {
+ debug(79, 3) ("storeCossOpen: %s memory miss - not reallocating (Current stripe : %d Object in stripe : %d)\n", SD->path, cs->curstripe, storeCossFilenoToStripe(cs, sio->swap_filen));
+ nf = storeCossMemOnlyAllocate(SD, e);
+ if (nf == -1) {
+ debug(79, 3) ("storeCossOpen: %s memory miss - reallocating because all membufs are in use\n", SD->path);
+ nf = storeCossAllocate(SD, e, COSS_ALLOC_REALLOC);
+ }
+ }
if (nf == -1) {
/* We have to clean up neatly .. */
coss_stats.open.fail++;
@@ -304,23 +409,36 @@
/* XXX XXX XXX Will squid call storeUnlink for this object? */
return NULL;
}
- /* Remove the object from its currently-allocated stripe */
- storeCossRemove(SD, e);
- storeCossNewPendingRelocate(cs, sio, sio->swap_filen, nf);
- sio->swap_filen = nf;
- cstate->flags.reloc = 1;
- /* Notify the upper levels that we've changed file number */
- sio->file_callback(sio->callback_data, 0, sio);
- /*
- * lock the new buffer so it doesn't get swapped out on us
- * this will get unlocked in storeCossClose
- */
- storeCossMemBufLock(SD, sio);
- /*
- * Do the index magic to keep the disk and memory LRUs identical
- * by adding the object into the link list on the current stripe
- */
- storeCossAdd(SD, e, cs->curstripe);
+ if (nf < cs->max_disk_nf) {
+ /* Remove the object from its currently-allocated stripe */
+ storeCossRemove(SD, e);
+ storeCossNewPendingRelocate(cs, sio, sio->swap_filen, nf);
+ sio->swap_filen = nf;
+ cstate->flags.reloc = 1;
+ /* Notify the upper levels that we've changed file number */
+ sio->file_callback(sio->callback_data, 0, sio);
+ /*
+ * lock the new buffer so it doesn't get swapped out on us
+ * this will get unlocked in storeCossClose
+ */
+ storeCossMemBufLock(SD, sio);
+ /*
+ * Do the index magic to keep the disk and memory LRUs identical
+ * by adding the object into the link list on the current stripe
+ */
+ storeCossAdd(SD, e, cs->curstripe);
+ } else {
+ /* Relocate the object in COSS, but not in other layers */
+ storeCossNewPendingRelocate(cs, sio, sio->swap_filen, nf);
+ sio->swap_filen = nf;
+ cstate->flags.reloc = 1;
+
+ /*
+ * lock the new buffer so it doesn't get swapped out on us
+ * this will get unlocked in storeCossClose
+ */
+ storeCossMemBufLock(SD, sio);
+ }
}
coss_stats.open.success++;
return sio;
@@ -357,7 +475,7 @@
assert(sio->read.callback_data == NULL);
sio->read.callback = callback;
sio->read.callback_data = callback_data;
- debug(79, 3) ("storeCossRead: %s: offset %ld\n", SD->path, (long int) offset);
+ debug(79, 3) ("storeCossRead: %s: file number %d offset %ld\n", SD->path, sio->swap_filen, (long int) offset);
sio->offset = offset;
cstate->flags.reading = 1;
if ((offset + size) > sio->st_size)
@@ -450,6 +568,37 @@
}
static void
+storeCossMemBufLockPending(CossPendingReloc * pr, CossMemBuf * t)
+{
+ assert(t->flags.dead == 0);
+ assert(pr->locked_membuf == NULL);
+ debug(79, 3) ("storeCossMemBufLockPending: locking %p, lockcount %d\n",
+ t, t->lockcount);
+ pr->locked_membuf = t;
+ t->lockcount++;
+}
+
+static void
+storeCossMemBufUnlockPending(CossPendingReloc * pr, CossInfo * cs)
+{
+ CossMemBuf *t = pr->locked_membuf;
+ if (NULL == t)
+ return;
+ assert(t->flags.dead == 0);
+ debug(79, 3) ("storeCossMemBufLockPending: unlocking %p, lockcount %d\n",
+ t, t->lockcount);
+ t->lockcount--;
+ pr->locked_membuf = NULL;
+
+ if (!t->flags.written) {
+ storeCossMaybeWriteMemBuf(t->SD, t);
+ } else {
+ /* cs->current_membuf may be invalid at this point */
+ storeCossMaybeFreeBuf(cs, t);
+ }
+}
+
+static void
storeCossMemBufLock(SwapDir * SD, storeIOState * sio)
{
CossMemBuf *t = storeCossFilenoToMembuf(SD, sio->swap_filen);
@@ -475,9 +624,12 @@
t, t->lockcount);
t->lockcount--;
cstate->locked_membuf = NULL;
- storeCossMaybeWriteMemBuf(SD, t);
- /* cs->current_membuf may be invalid at this point */
- storeCossMaybeFreeBuf(cs, t);
+ if (!t->flags.written) {
+ storeCossMaybeWriteMemBuf(SD, t);
+ } else {
+ /* cs->current_membuf may be invalid at this point */
+ storeCossMaybeFreeBuf(cs, t);
+ }
}
static void
@@ -532,7 +684,8 @@
storeCossWriteMemBuf(SwapDir * SD, CossMemBuf * t)
{
CossInfo *cs = (CossInfo *) SD->fsdata;
- coss_stats.stripe_write.ops++;
+ int cur_load_interval = (squid_curtime / cs->load_interval) % 2;
+ int prev_load_interval = ((squid_curtime + cs->load_interval) / cs->load_interval) % 2;
assert(t->flags.dead == 0);
debug(79, 3) ("storeCossWriteMemBuf: %p: offset %ld, len %ld\n", t,
(long int) t->diskstart, (long int) (t->diskend - t->diskstart));
@@ -545,26 +698,38 @@
* before the objects underneath the membufs stripe were purged and there
* is still a pending relocate for it. Its a slim chance but it might happen.
*/
- assert(t->stripe < cs->numstripes);
- if (cs->stripes[t->stripe].pending_relocs > 0) {
- debug(79, 1) ("WARNING: %s: One or more pending relocate (reads) from stripe %d are queued - and I'm now writing over that part of the disk. This may result in object data corruption!\n", SD->path, t->stripe);
- }
- /*
- * normally nothing should have this node locked here - but between the time
- * we call a_file_write and the IO completes someone might have snuck in and
- * attached itself somehow. This is why there's a distinction between "written"
- * and "writing". Read the rest of the code for more details.
- */
+ if (!(t->flags.memonly)) {
+ coss_stats.stripe_write.ops++;
+ assert(t->stripe < cs->numstripes);
+ if (cs->stripes[t->stripe].pending_relocs > 0) {
+ debug(79, 1) ("WARNING: %s: One or more pending relocate (reads) from stripe %d are queued - and I'm now writing over that part of the disk. This may result in object data corruption!\n", SD->path, t->stripe);
+ }
+ /* Update load stats */
+ cs->loadcalc[cur_load_interval] += 1;
+ cs->loadcalc[prev_load_interval] = 0;
+
+ /*
+ * normally nothing should have this node locked here - but between the time
+ * we call a_file_write and the IO completes someone might have snuck in and
+ * attached itself somehow. This is why there's a distinction between "written"
+ * and "writing". Read the rest of the code for more details.
+ */
#if USE_AUFSOPS
- /* XXX The last stripe, for now, ain't the coss stripe size for some reason */
- /* XXX This may cause problems later on; worry about figuring it out later on */
- //assert(t->diskend - t->diskstart == COSS_MEMBUF_SZ);
- debug(79, 3) ("aioWrite: FD %d: disk start: %llu, size %llu\n", cs->fd, (long long int) t->diskstart, (long long int) t->diskend - t->diskstart);
- aioWrite(cs->fd, t->diskstart, &(t->buffer[0]), t->diskend - t->diskstart, storeCossWriteMemBufDone, t, NULL);
+ /* XXX The last stripe, for now, ain't the coss stripe size for some reason */
+ /* XXX This may cause problems later on; worry about figuring it out later on */
+ //assert(t->diskend - t->diskstart == COSS_MEMBUF_SZ);
+ debug(79, 3) ("aioWrite: FD %d: disk start: %llu, size %llu\n", cs->fd, (long long int) t->diskstart, (long long int) t->diskend - t->diskstart);
+ aioWrite(cs->fd, t->diskstart, &(t->buffer[0]), t->diskend - t->diskstart, storeCossWriteMemBufDone, t, NULL);
#else
- a_file_write(&cs->aq, cs->fd, t->diskstart, &t->buffer,
- t->diskend - t->diskstart, storeCossWriteMemBufDone, t, NULL);
+ a_file_write(&cs->aq, cs->fd, t->diskstart, &t->buffer,
+ t->diskend - t->diskstart, storeCossWriteMemBufDone, t, NULL);
#endif
+ } else {
+ /* No need to write, just mark as written and free */
+ t->flags.written = 1;
+ t->flags.writing = 0;
+ storeCossMaybeFreeBuf(cs, t);
+ }
}
/*
@@ -587,6 +752,15 @@
* call the asyncio disk completion handler.)
*/
if (mb->lockcount == 0 && mb->flags.written == 1) {
+ /* We need to wait until here to mark the membuf as
+ * free so we can re-alocate it
+ */
+ if (mb->flags.memonly) {
+ assert(cs->memstripes[mb->stripe].membuf == mb);
+ cs->memstripes[mb->stripe].membuf = NULL;
+ } else {
+ cs->numfullstripes--;
+ }
debug(79, 3) ("storeCossMaybeFreeBuf: %p: lockcount = 0, written = 1: marking dead\n", mb);
mb->flags.dead = 1;
dlinkDelete(&mb->node, &cs->membufs);
@@ -655,6 +829,49 @@
storeCossMaybeFreeBuf(cs, t);
}
+static CossMemBuf *
+storeCossCreateMemOnlyBuf(SwapDir * SD)
+{
+ CossMemBuf *newmb;
+ CossInfo *cs = (CossInfo *) SD->fsdata;
+ off_t start;
+ int stripe;
+ static time_t last_warn = 0;
+
+ /* TODO: Maybe make this a simple search for a free membuf */
+ for (stripe = 0; stripe < cs->nummemstripes; stripe++) {
+ if (cs->memstripes[stripe].membuf == NULL)
+ break;
+ }
+ if (stripe >= cs->nummemstripes) {
+ if (last_warn + 15 < squid_curtime) {
+ debug(79, 1) ("storeCossCreateMemOnlyBuf: no free membufs. You may need to increase the value of membufs on the %s cache_dir\n", SD->path);
+ last_warn = squid_curtime;
+ }
+ return NULL;
+ }
+ cs->curmemstripe = stripe;
+
+ start = (off_t) stripe *COSS_MEMBUF_SZ;
+ newmb = cbdataAlloc(CossMemBuf);
+
+ cs->memstripes[stripe].membuf = newmb;
+ newmb->diskstart = ((off_t) SD->max_size << 10) + start;
+ newmb->stripe = stripe;
+ newmb->diskend = newmb->diskstart + COSS_MEMBUF_SZ;
+ newmb->flags.full = 0;
+ newmb->flags.writing = 0;
+ newmb->flags.memonly = 1;
+ newmb->lockcount = 0;
+ newmb->numobjs = 0;
+ newmb->SD = SD;
+
+ dlinkAdd(newmb, &newmb->node, &cs->membufs);
+
+ coss_stats.stripes++;
+ return newmb;
+}
+
/*
* This creates a memory buffer but assumes its going to be at the end
* of the "LRU" and thusly will delete expire objects which appear under
@@ -670,8 +887,16 @@
CossInfo *cs = (CossInfo *) SD->fsdata;
off_t start = (off_t) stripe * COSS_MEMBUF_SZ;
off_t o;
+ static time_t last_warn = 0;
assert(start >= 0);
+ if (cs->numfullstripes >= cs->maxfullstripes) {
+ if (last_warn + 15 < squid_curtime) {
+ debug(79, 1) ("storeCossCreateMemBuf: Maximum number of full buffers reached on %s. You may need to increase the maxfullbuffers option for this cache_dir\n", SD->path);
+ last_warn = squid_curtime;
+ }
+ return NULL;
+ }
/* No, we shouldn't ever try to create a membuf if we haven't freed the one on
* this stripe. Grr */
assert(cs->stripes[stripe].membuf == NULL);
@@ -742,6 +967,12 @@
newmb = storeCossCreateMemBuf(sd, 0, -1, NULL);
assert(!cs->current_membuf);
cs->current_membuf = newmb;
+
+ newmb = storeCossCreateMemOnlyBuf(sd);
+ assert(!cs->current_memonly_membuf);
+ cs->current_memonly_membuf = newmb;
+
+ cs->current_memonly_offset = cs->current_memonly_membuf->diskstart;
}
/*
@@ -802,6 +1033,7 @@
storeCossNewPendingRelocate(CossInfo * cs, storeIOState * sio, sfileno original_filen, sfileno new_filen)
{
CossPendingReloc *pr;
+ CossMemBuf *membuf;
char *p;
off_t disk_offset;
int stripe;
@@ -824,8 +1056,12 @@
cs->stripes[stripe].pending_relocs++;
/* And now; we begin the IO */
- p = storeCossMemPointerFromDiskOffset(cs, storeCossFilenoToDiskOffset(new_filen, cs), NULL);
+ p = storeCossMemPointerFromDiskOffset(cs, storeCossFilenoToDiskOffset(new_filen, cs), &membuf);
pr->p = p;
+
+ /* Lock the destination membuf */
+ storeCossMemBufLockPending(pr, membuf);
+
disk_offset = storeCossFilenoToDiskOffset(original_filen, cs);
debug(79, 3) ("COSS Pending Relocate: size %" PRINTF_OFF_T ", disk_offset %llu\n", (squid_off_t) sio->e->swap_file_sz, (long long int) disk_offset);
#if USE_AUFSOPS
@@ -928,6 +1164,8 @@
/* XXX again, this shouldn't be here (find the dlinkAddTail() in storeCossKickReadOp); these should
* be abstracted out. */
}
+ /* Unlock (and possibly write/free) the destination membuf */
+ storeCossMemBufUnlockPending(pr, cs);
/* Good, now we can delete it */
cbdataUnlock(pr);
cbdataFree(pr);
@@ -1038,11 +1276,12 @@
static void
membufsPrint(StoreEntry * e, CossMemBuf * t, const char *prefix)
{
- storeAppendPrintf(e, "%s: %d, lockcount: %d, numobjects %d, flags: %s,%s,%s\n",
+ storeAppendPrintf(e, "%s: %d, lockcount: %d, numobjects %d, flags: %s,%s,%s,%s\n",
prefix, t->stripe, t->lockcount, t->numobjs,
t->flags.full ? "FULL" : "NOTFULL",
t->flags.writing ? "WRITING" : "NOTWRITING",
- t->flags.written ? "WRITTEN" : "NOTWRITTEN");
+ t->flags.written ? "WRITTEN" : "NOTWRITTEN",
+ t->flags.memonly ? "MEMONLY" : "DISK");
}
void
diff -ruN squid-2.6.STABLE2/src/fs/diskd/store_dir_diskd.c squid-2.6.STABLE3/src/fs/diskd/store_dir_diskd.c
--- squid-2.6.STABLE2/src/fs/diskd/store_dir_diskd.c Sun Jul 30 17:47:14 2006
+++ squid-2.6.STABLE3/src/fs/diskd/store_dir_diskd.c Wed Aug 2 20:31:12 2006
@@ -1,6 +1,6 @@
/*
- * $Id: store_dir_diskd.c,v 1.81 2006/07/30 23:47:14 hno Exp $
+ * $Id: store_dir_diskd.c,v 1.83 2006/08/03 02:31:12 adrian Exp $
*
* DEBUG: section 47 Store Directory Routines
* AUTHOR: Duane Wessels
@@ -1751,6 +1751,7 @@
void
storeDiskdDirMaintain(SwapDir * SD)
{
+ diskdinfo_t *diskdinfo = SD->fsdata;
StoreEntry *e = NULL;
int removed = 0;
int max_scan;
@@ -1772,7 +1773,7 @@
debug(20, 3) ("storeMaintainSwapSpace: f=%f, max_scan=%d, max_remove=%d\n", f, max_scan, max_remove);
walker = SD->repl->PurgeInit(SD->repl, max_scan);
while (1) {
- if (SD->cur_size < SD->low_size)
+ if (SD->cur_size < SD->low_size && diskdinfo->map->n_files_in_map < FILEMAP_MAX)
break;
if (removed >= max_remove)
break;
@@ -1812,7 +1813,7 @@
/* the parse function guarantees magic2 is positivie */
if (diskdinfo->away >= diskdinfo->magic1)
return -1;
- return diskdinfo->away * 1000 / diskdinfo->magic2;
+ return DISKD_LOAD_BASE + (diskdinfo->away * DISKD_LOAD_QUEUE_WEIGHT / diskdinfo->magic2);
}
/*
@@ -1961,6 +1962,7 @@
storeAppendPrintf(sentry, "Current Size: %d KB\n", SD->cur_size);
storeAppendPrintf(sentry, "Percent Used: %0.2f%%\n",
100.0 * SD->cur_size / SD->max_size);
+ storeAppendPrintf(sentry, "Current load metric: %d / %d\n", storeDiskdDirCheckLoadAv(SD, ST_OP_CREATE), MAX_LOAD_VALUE);
storeAppendPrintf(sentry, "Filemap bits in use: %d of %d (%d%%)\n",
diskdinfo->map->n_files_in_map, diskdinfo->map->max_n_files,
percent(diskdinfo->map->n_files_in_map, diskdinfo->map->max_n_files));
diff -ruN squid-2.6.STABLE2/src/fs/ufs/store_dir_ufs.c squid-2.6.STABLE3/src/fs/ufs/store_dir_ufs.c
--- squid-2.6.STABLE2/src/fs/ufs/store_dir_ufs.c Sun Jul 30 17:27:05 2006
+++ squid-2.6.STABLE3/src/fs/ufs/store_dir_ufs.c Wed Aug 2 20:31:13 2006
@@ -1,6 +1,6 @@
/*
- * $Id: store_dir_ufs.c,v 1.59 2006/07/30 23:27:05 hno Exp $
+ * $Id: store_dir_ufs.c,v 1.61 2006/08/03 02:31:13 adrian Exp $
*
* DEBUG: section 47 Store Directory Routines
* AUTHOR: Duane Wessels
@@ -1554,6 +1554,7 @@
void
storeUfsDirMaintain(SwapDir * SD)
{
+ ufsinfo_t *ufsinfo = SD->fsdata;
StoreEntry *e = NULL;
int removed = 0;
int max_scan;
@@ -1575,7 +1576,7 @@
debug(47, 3) ("storeMaintainSwapSpace: f=%f, max_scan=%d, max_remove=%d\n", f, max_scan, max_remove);
walker = SD->repl->PurgeInit(SD->repl, max_scan);
while (1) {
- if (SD->cur_size < SD->low_size)
+ if (SD->cur_size < SD->low_size && ufsinfo->map->n_files_in_map < FILEMAP_MAX)
break;
if (removed >= max_remove)
break;
@@ -1612,7 +1613,7 @@
storeUfsDirCheckLoadAv(SwapDir * SD, store_op_t op)
{
ufsinfo_t *ufsinfo = SD->fsdata;
- return 500 + ufsinfo->open_files / 2;
+ return UFS_LOAD_BASE + ufsinfo->open_files / 2;
}
/*
@@ -1714,6 +1715,7 @@
storeAppendPrintf(sentry, "Current Size: %d KB\n", SD->cur_size);
storeAppendPrintf(sentry, "Percent Used: %0.2f%%\n",
100.0 * SD->cur_size / SD->max_size);
+ storeAppendPrintf(sentry, "Current load metric: %d / %d\n", storeUfsDirCheckLoadAv(SD, ST_OP_CREATE), MAX_LOAD_VALUE);
storeAppendPrintf(sentry, "Filemap bits in use: %d of %d (%d%%)\n",
ufsinfo->map->n_files_in_map, ufsinfo->map->max_n_files,
percent(ufsinfo->map->n_files_in_map, ufsinfo->map->max_n_files));
diff -ruN squid-2.6.STABLE2/src/ipcache.c squid-2.6.STABLE3/src/ipcache.c
--- squid-2.6.STABLE2/src/ipcache.c Wed May 24 20:51:58 2006
+++ squid-2.6.STABLE3/src/ipcache.c Tue Aug 15 18:27:20 2006
@@ -1,6 +1,6 @@
/*
- * $Id: ipcache.c,v 1.244 2006/05/25 02:51:58 hno Exp $
+ * $Id: ipcache.c,v 1.245 2006/08/16 00:27:20 hno Exp $
*
* DEBUG: section 14 IP Cache
* AUTHOR: Harvest Derived
@@ -363,7 +363,10 @@
if (ttl == 0 || ttl > answers[k].ttl)
ttl = answers[k].ttl;
}
- i->addrs.count = (unsigned char) na;
+ if (na < 256)
+ i->addrs.count = (unsigned char) na;
+ else
+ i->addrs.count = 255;
if (ttl == 0 || ttl > Config.positiveDnsTtl)
ttl = Config.positiveDnsTtl;
if (ttl < Config.negativeDnsTtl)
diff -ruN squid-2.6.STABLE2/src/main.c squid-2.6.STABLE3/src/main.c
--- squid-2.6.STABLE2/src/main.c Sun Jul 30 17:27:03 2006
+++ squid-2.6.STABLE3/src/main.c Tue Aug 15 13:27:28 2006
@@ -1,6 +1,6 @@
/*
- * $Id: main.c,v 1.387 2006/07/30 23:27:03 hno Exp $
+ * $Id: main.c,v 1.388 2006/08/15 19:27:28 hno Exp $
*
* DEBUG: section 1 Startup and Main Loop
* AUTHOR: Harvest Derived
@@ -552,6 +552,7 @@
debug(1, 1) ("Windows sockets initialized\n");
#endif
+ comm_select_postinit();
if (!configured_once)
disk_init(); /* disk_init must go before ipcache_init() */
ipcache_init();
diff -ruN squid-2.6.STABLE2/src/protos.h squid-2.6.STABLE3/src/protos.h
--- squid-2.6.STABLE2/src/protos.h Sun Jul 30 17:27:03 2006
+++ squid-2.6.STABLE3/src/protos.h Tue Aug 15 18:54:16 2006
@@ -1,6 +1,6 @@
/*
- * $Id: protos.h,v 1.507 2006/07/30 23:27:03 hno Exp $
+ * $Id: protos.h,v 1.510 2006/08/16 00:54:16 hno Exp $
*
*
* SQUID Web Proxy Cache http://www.squid-cache.org/
@@ -185,6 +185,7 @@
* comm_select.c
*/
extern void comm_select_init(void);
+extern void comm_select_postinit(void);
extern void comm_select_shutdown(void);
extern int comm_select(int);
extern void commUpdateEvents(int fd);
@@ -192,6 +193,7 @@
extern void commUpdateReadHandler(int, PF *, void *);
extern void commUpdateWriteHandler(int, PF *, void *);
extern void comm_quick_poll_required(void);
+extern void comm_select_status(StoreEntry *);
extern void packerToStoreInit(Packer * p, StoreEntry * e);
extern void packerToMemInit(Packer * p, MemBuf * mb);
@@ -603,7 +605,7 @@
/* unfirtunate hack to test if the buffer has been Init()ialized */
extern int memBufIsNull(MemBuf * mb);
/* calls memcpy, appends exactly size bytes, extends buffer if needed */
-extern void memBufAppend(MemBuf * mb, const char *buf, int size);
+extern void memBufAppend(MemBuf * mb, const void *buf, int size);
/* calls snprintf, extends buffer if needed */
#if STDC_HEADERS
extern void
diff -ruN squid-2.6.STABLE2/src/refresh.c squid-2.6.STABLE3/src/refresh.c
--- squid-2.6.STABLE2/src/refresh.c Fri Jul 28 14:49:09 2006
+++ squid-2.6.STABLE3/src/refresh.c Fri Aug 18 15:06:04 2006
@@ -1,6 +1,6 @@
/*
- * $Id: refresh.c,v 1.61 2006/07/28 20:49:09 hno Exp $
+ * $Id: refresh.c,v 1.62 2006/08/18 21:06:04 hno Exp $
*
* DEBUG: section 22 Refresh Calculation
* AUTHOR: Harvest Derived
@@ -168,6 +168,13 @@
debug(22, 3) ("STALE: age %d > max %d \n", (int) age, (int) R->max);
sf->max = 1;
return (age - R->max);
+ }
+ if (check_time < entry->timestamp) {
+ debug(22, 1) ("STALE: Entry's timestamp greater than check time. Clock going backwards?\n");
+ debug(22, 1) ("\tcheck_time:\t%s\n", mkrfc1123(check_time));
+ debug(22, 1) ("\tentry->timestamp:\t%s\n", mkrfc1123(entry->timestamp));
+ debug(22, 1) ("\tstaleness:\t%ld\n", (long int) entry->timestamp - check_time);
+ return (entry->timestamp - check_time);
}
/*
* Try the last-modified factor algorithm.
diff -ruN squid-2.6.STABLE2/src/stat.c squid-2.6.STABLE3/src/stat.c
--- squid-2.6.STABLE2/src/stat.c Sat Jul 29 11:35:31 2006
+++ squid-2.6.STABLE3/src/stat.c Tue Aug 15 13:27:28 2006
@@ -1,6 +1,6 @@
/*
- * $Id: stat.c,v 1.373 2006/07/29 17:35:31 serassio Exp $
+ * $Id: stat.c,v 1.374 2006/08/15 19:27:28 hno Exp $
*
* DEBUG: section 18 Cache Manager Statistics
* AUTHOR: Harvest Derived
@@ -659,6 +659,7 @@
RESERVED_FD);
storeAppendPrintf(sentry, "\tStore Disk files open: %4d\n",
store_open_disk_fd);
+ comm_select_status(sentry);
storeAppendPrintf(sentry, "Internal Data Structures:\n");
storeAppendPrintf(sentry, "\t%6d StoreEntries\n",
diff -ruN squid-2.6.STABLE2/src/store.c squid-2.6.STABLE3/src/store.c
--- squid-2.6.STABLE2/src/store.c Sun Jul 16 20:32:00 2006
+++ squid-2.6.STABLE3/src/store.c Tue Aug 15 18:31:29 2006
@@ -1,6 +1,6 @@
/*
- * $Id: store.c,v 1.565 2006/07/17 02:32:00 hno Exp $
+ * $Id: store.c,v 1.566 2006/08/16 00:31:29 hno Exp $
*
* DEBUG: section 20 Storage Manager
* AUTHOR: Harvest Derived
@@ -418,18 +418,21 @@
{
AddVaryState *state = data;
debug(11, 2) ("free_AddVaryState: %p\n", data);
- if (!state->done && state->key) {
- storeAppendPrintf(state->e, "Key: %s\n", state->key);
- if (state->accept_encoding)
- storeAppendPrintf(state->e, "Accept-Encoding: %s\n", state->accept_encoding);
- if (state->etag)
- storeAppendPrintf(state->e, "ETag: %s\n", state->etag);
- storeAppendPrintf(state->e, "VaryData: %s\n", state->vary_headers);
+ if (!EBIT_TEST(state->e->flags, ENTRY_ABORTED)) {
+ storeBuffer(state->e);
+ if (!state->done && state->key) {
+ storeAppendPrintf(state->e, "Key: %s\n", state->key);
+ if (state->accept_encoding)
+ storeAppendPrintf(state->e, "Accept-Encoding: %s\n", state->accept_encoding);
+ if (state->etag)
+ storeAppendPrintf(state->e, "ETag: %s\n", state->etag);
+ storeAppendPrintf(state->e, "VaryData: %s\n", state->vary_headers);
+ }
+ storeTimestampsSet(state->e);
+ storeComplete(state->e);
+ storeTimestampsSet(state->e);
+ storeBufferFlush(state->e);
}
- storeBufferFlush(state->e);
- storeTimestampsSet(state->e);
- storeComplete(state->e);
- storeTimestampsSet(state->e);
storeUnlockObject(state->e);
state->e = NULL;
if (state->sc) {
@@ -539,6 +542,12 @@
cbdataFree(state);
return;
}
+ if (EBIT_TEST(state->e->flags, ENTRY_ABORTED)) {
+ debug(11, 1) ("storeAddVaryReadOld: New index aborted at %d (%d)\n", (int) state->seen_offset, (int) size);
+ cbdataFree(state);
+ return;
+ }
+ storeBuffer(state->e);
if (state->seen_offset != 0) {
state->seen_offset = state->seen_offset + size;
} else {
@@ -661,6 +670,7 @@
if (l)
memmove(state->buf, p, l);
debug(11, 3) ("storeAddVaryReadOld: %p seen_offset=%" PRINTF_OFF_T " buf_offset=%d\n", data, state->seen_offset, (int) state->buf_offset);
+ storeBufferFlush(state->e);
storeClientCopy(state->sc, state->oe,
state->seen_offset,
state->seen_offset,
@@ -704,7 +714,6 @@
storeSetPublicKey(state->e);
storeBuffer(state->e);
httpReplySwapOut(state->e->mem_obj->reply, state->e);
- storeBufferFlush(state->e);
if (state->oe) {
/* Here we need to tack on the old etag/vary information, and we should
* merge, clean up etc
diff -ruN squid-2.6.STABLE2/src/store_swapout.c squid-2.6.STABLE3/src/store_swapout.c
--- squid-2.6.STABLE2/src/store_swapout.c Thu Jun 1 18:07:40 2006
+++ squid-2.6.STABLE3/src/store_swapout.c Mon Aug 7 09:03:28 2006
@@ -1,6 +1,6 @@
/*
- * $Id: store_swapout.c,v 1.94 2006/06/02 00:07:40 hno Exp $
+ * $Id: store_swapout.c,v 1.95 2006/08/07 15:03:28 hno Exp $
*
* DEBUG: section 20 Storage Manager Swapout Functions
* AUTHOR: Duane Wessels
@@ -243,8 +243,13 @@
assert(mem->inmem_lo == 0);
if (storeCheckCachable(e))
storeSwapOutStart(e);
- else
+ else {
+ /* Now that we know the data is not cachable, free the memory
+ * to make sure the forwarding code does not defer the connection
+ */
+ storeSwapOutMaintainMemObject(e);
return;
+ }
/* ENTRY_CACHABLE will be cleared and we'll never get here again */
}
if (NULL == mem->swapout.sio)
diff -ruN squid-2.6.STABLE2/src/structs.h squid-2.6.STABLE3/src/structs.h
--- squid-2.6.STABLE2/src/structs.h Sun Jul 30 17:27:03 2006
+++ squid-2.6.STABLE3/src/structs.h Sat Aug 12 07:33:45 2006
@@ -1,6 +1,6 @@
/*
- * $Id: structs.h,v 1.496 2006/07/30 23:27:03 hno Exp $
+ * $Id: structs.h,v 1.497 2006/08/12 13:33:45 adrian Exp $
*
*
* SQUID Web Proxy Cache http://www.squid-cache.org/
@@ -843,6 +843,16 @@
FREE *free_func;
};
+struct _CommWriteStateData {
+ int valid;
+ char *buf;
+ size_t size;
+ size_t offset;
+ CWCB *handler;
+ void *handler_data;
+ FREE *free_func;
+};
+
/* ETag support is rudimantal;
* this struct is likely to change
@@ -898,7 +908,7 @@
close_handler *close_handler; /* linked list */
DEFER *defer_check; /* check if we should defer read */
void *defer_data;
- CommWriteStateData *rwstate; /* State data for comm_write */
+ struct _CommWriteStateData rwstate; /* State data for comm_write */
READ_HANDLER *read_method;
WRITE_HANDLER *write_method;
#if USE_SSL
@@ -1911,15 +1921,6 @@
unsigned int ignore_auth:1;
#endif
} flags;
-};
-
-struct _CommWriteStateData {
- char *buf;
- size_t size;
- size_t offset;
- CWCB *handler;
- void *handler_data;
- FREE *free_func;
};
struct _ErrorState {
diff -ruN squid-2.6.STABLE2/tools/cossdump.c squid-2.6.STABLE3/tools/cossdump.c
--- squid-2.6.STABLE2/tools/cossdump.c Sat Jul 8 05:04:46 2006
+++ squid-2.6.STABLE3/tools/cossdump.c Sun Aug 6 11:12:56 2006
@@ -1,10 +1,27 @@
+#include "config.h"
+
+#if HAVE_INTTYPES_H
+#include
+#endif
+#if HAVE_STDIO_H
#include
+#endif
+#if HAVE_STDLIB_H
#include
+#endif
+#if HAVE_UNISTD_H
#include
+#endif
#include
+#if HAVE_STRING_H
#include
+#endif
+#if HAVE_SYS_TYPES_H
#include
+#endif
+#if HAVE_FCNTL_H
#include
+#endif
#include "../src/defines.h"
#include "../src/enums.h"
@@ -25,10 +42,19 @@
#define MEM_TLV sizeof(tlv)
#define memAllocate(a) malloc(a)
#define memFree(a, b) free(a)
-#define xmemcpy(a, b, c) memcpy(a, b, c)
#define xmalloc(a) malloc(a)
#define xfree(a) free(a)
+#ifndef PRId64
+#ifdef _SQUID_MSWIN_ /* Windows native port using MSVCRT */
+#define PRId64 "I64d"
+#elif SIZEOF_INT64_T > SIZEOF_LONG
+#define PRId64 "lld"
+#else
+#define PRId64 "ld"
+#endif
+#endif
+
#if SIZEOF_INT64_T > SIZEOF_LONG && HAVE_STRTOLL
typedef int64_t squid_off_t;
#define SIZEOF_SQUID_OFF_T SIZEOF_INT64_T
@@ -153,66 +179,65 @@
static void
parse_stripe(int stripeid, char *buf, int len)
{
- int j = 0;
- int bl = 0;
- tlv *t, *tlv_list;
- int64_t *l;
- int tmp;
-
- while (j < len) {
- l = NULL;
- bl = 0;
- tlv_list = storeSwapMetaUnpack(&buf[j], &bl);
- if (tlv_list == NULL) {
- printf(" Object: NULL\n");
- return;
- }
- printf(" Object: (filen %d) hdr size %d\n", j / BLOCKSIZE + (stripeid * STRIPESIZE / BLOCKSIZE), bl);
- for (t = tlv_list; t; t = t->next) {
- switch(t->type) {
- case STORE_META_URL:
- /* XXX Is this OK? Is the URL guaranteed to be \0 terminated? */
- printf(" URL: %s\n", (char *)t->value);
- break;
- case STORE_META_OBJSIZE:
- l = t->value;
- printf("Size: %" PRINTF_OFF_T " (len %d)\n", *l, t->length);
- break;
- }
- }
- if (l == NULL) {
- printf(" STRIPE: Completed, got an object with no size\n");
- return;
- }
- j = j + *l + bl;
- /* And now, the blocksize! */
- tmp = j / BLOCKSIZE;
- tmp = (tmp+1) * BLOCKSIZE;
- j = tmp;
- }
+ int j = 0;
+ int bl = 0;
+ tlv *t, *tlv_list;
+ int64_t *l;
+ int tmp;
+
+ while (j < len) {
+ l = NULL;
+ bl = 0;
+ tlv_list = storeSwapMetaUnpack(&buf[j], &bl);
+ if (tlv_list == NULL) {
+ printf(" Object: NULL\n");
+ return;
+ }
+ printf(" Object: (filen %d) hdr size %d\n", j / BLOCKSIZE + (stripeid * STRIPESIZE / BLOCKSIZE), bl);
+ for (t = tlv_list; t; t = t->next) {
+ switch (t->type) {
+ case STORE_META_URL:
+ /* XXX Is this OK? Is the URL guaranteed to be \0 terminated? */
+ printf(" URL: %s\n", (char *) t->value);
+ break;
+ case STORE_META_OBJSIZE:
+ l = t->value;
+ printf("Size: %" PRINTF_OFF_T " (len %d)\n", *l, t->length);
+ break;
+ }
+ }
+ if (l == NULL) {
+ printf(" STRIPE: Completed, got an object with no size\n");
+ return;
+ }
+ j = j + *l + bl;
+ /* And now, the blocksize! */
+ tmp = j / BLOCKSIZE;
+ tmp = (tmp + 1) * BLOCKSIZE;
+ j = tmp;
+ }
}
int
main(int argc, char *argv[])
{
- int fd;
- char buf[STRIPESIZE];
- int i = 0, len;
-
- if (argc < 2) {
- printf("Usage: %s \n", argv[0]);
- exit(1);
- }
-
- fd = open(argv[1], O_RDONLY);
- if (fd < 0) {
- perror("open");
- exit(1);
- }
- while ((len = read(fd, buf, STRIPESIZE)) > 0) {
- printf("STRIPE: %d (len %d)\n", i, len);
- parse_stripe(i, buf, len);
- i++;
- }
- return 0;
+ int fd;
+ char buf[STRIPESIZE];
+ int i = 0, len;
+
+ if (argc < 2) {
+ printf("Usage: %s \n", argv[0]);
+ exit(1);
+ }
+ fd = open(argv[1], O_RDONLY);
+ if (fd < 0) {
+ perror("open");
+ exit(1);
+ }
+ while ((len = read(fd, buf, STRIPESIZE)) > 0) {
+ printf("STRIPE: %d (len %d)\n", i, len);
+ parse_stripe(i, buf, len);
+ i++;
+ }
+ return 0;
}