Merge pull request #316 from NLnetLabs/bugfix/mitigate-some-defunct-processes

wtoorop · web-flow · commit ead1fb38303f · 2024-02-26T12:12:59.000+01:00
Bugfix/mitigate some &lt;defunct&gt; processes
diff --git a/server.c b/server.c
@@ -2311,6 +2311,78 @@ reload_process_tasks(struct nsd* nsd, udb_ptr* last_task, int cmdsocket)
 
 void server_verify(struct nsd *nsd, int cmdsocket);
 
+struct quit_sync_event_data {
+	struct event_base* base;
+	size_t read;
+	union {
+		uint8_t buf[sizeof(sig_atomic_t)];
+		sig_atomic_t cmd;
+	} to_read;
+};
+
+static void server_reload_handle_sigchld(int sig, short event,
+		void* ATTR_UNUSED(arg))
+{
+	assert(sig == SIGCHLD);
+	assert(event & EV_SIGNAL);
+
+	/* reap the exited old-serve child(s) */
+	while(waitpid(-1, NULL, WNOHANG) > 0) {
+		/* pass */
+	}
+}
+
+static void server_reload_handle_quit_sync_ack(int cmdsocket, short event,
+		void* arg)
+{
+	struct quit_sync_event_data* cb_data =
+		(struct quit_sync_event_data*)arg;
+	ssize_t r;
+
+	if(event & EV_TIMEOUT) {
+		sig_atomic_t cmd = NSD_QUIT_SYNC;
+
+		DEBUG(DEBUG_IPC,1, (LOG_INFO, "reload: ipc send quit to main"));
+		if (!write_socket(cmdsocket, &cmd, sizeof(cmd))) {
+			log_msg(LOG_ERR, "problems sending command from "
+				"reload to old-main: %s", strerror(errno));
+		}
+		/* Wait for cmdsocket to become readable or for next timeout,
+		 * (this works because event is added EV_TIMEOUT|EV_PERSIST).
+		 */
+		return;
+	}
+	assert(event & EV_READ);
+	assert(cb_data->read < sizeof(cb_data->to_read.cmd));
+
+	r = read(cmdsocket, cb_data->to_read.buf + cb_data->read,
+			sizeof(cb_data->to_read.cmd) - cb_data->read);
+	if(r == 0) {
+		log_msg(LOG_ERR, "reload: old-main quit during quit sync");
+		cb_data->to_read.cmd = NSD_RELOAD;
+
+	} else if(r == -1) {
+		if(errno == EAGAIN || errno == EINTR)
+			return;
+
+		log_msg(LOG_ERR, "reload: could not wait for parent to quit: "
+			"%s", strerror(errno));
+		cb_data->to_read.cmd = NSD_RELOAD;
+
+	} else if (cb_data->read + r  < sizeof(cb_data->to_read.cmd)) {
+		/* More to read */
+		cb_data->read += r;
+		return;
+
+	} else {
+		assert(cb_data->read + r == sizeof(cb_data->to_read.cmd));
+		DEBUG(DEBUG_IPC,1, (LOG_INFO, "reload: ipc reply main %d",
+					(int)cb_data->to_read.cmd));
+	}
+	/* Done */
+	event_base_loopexit(cb_data->base, NULL);
+}
+
 /*
  * Reload the database, stop parent, re-fork children and continue.
  * as server_main.
@@ -2320,13 +2392,16 @@ server_reload(struct nsd *nsd, region_type* server_region, netio_type* netio,
 	int cmdsocket)
 {
 	pid_t mypid;
-	sig_atomic_t cmd = NSD_QUIT_SYNC;
-	int ret;
+	sig_atomic_t cmd;
 	udb_ptr last_task;
 	struct sigaction old_sigchld, ign_sigchld;
 	struct radnode* node;
 	zone_type* zone;
 	enum soainfo_hint hint;
+	struct quit_sync_event_data cb_data;
+	struct event signal_event, cmd_event;
+	struct timeval reload_sync_timeout;
+
 	/* ignore SIGCHLD from the previous server_main that used this pid */
 	memset(&ign_sigchld, 0, sizeof(ign_sigchld));
 	ign_sigchld.sa_handler = SIG_IGN;
@@ -2430,7 +2505,7 @@ server_reload(struct nsd *nsd, region_type* server_region, netio_type* netio,
 		exit(1);
 	}
 
-	/* if the parent has quit, we must quit too, poll the fd for cmds */
+	/* if the old-main has quit, we must quit too, poll the fd for cmds */
 	if(block_read(nsd, cmdsocket, &cmd, sizeof(cmd), 0) == sizeof(cmd)) {
 		DEBUG(DEBUG_IPC,1, (LOG_INFO, "reload: ipc command from main %d", (int)cmd));
 		if(cmd == NSD_QUIT) {
@@ -2440,34 +2515,55 @@ server_reload(struct nsd *nsd, region_type* server_region, netio_type* netio,
 		}
 	}
 
-	/* Send quit command to parent: blocking, wait for receipt. */
-	do {
-		DEBUG(DEBUG_IPC,1, (LOG_INFO, "reload: ipc send quit to main"));
-		cmd = NSD_QUIT_SYNC;
-		if (!write_socket(cmdsocket, &cmd, sizeof(cmd)))
-		{
-			log_msg(LOG_ERR, "problems sending command from reload to oldnsd: %s",
-				strerror(errno));
-		}
-		/* blocking: wait for parent to really quit. (it sends RELOAD as ack) */
-		DEBUG(DEBUG_IPC,1, (LOG_INFO, "reload: ipc wait for ack main"));
-		ret = block_read(nsd, cmdsocket, &cmd, sizeof(cmd),
-			RELOAD_SYNC_TIMEOUT);
-		if(ret == -2) {
-			DEBUG(DEBUG_IPC, 1, (LOG_ERR, "reload timeout QUITSYNC. retry"));
-		}
-	} while (ret == -2);
-	if(ret == -1) {
-		log_msg(LOG_ERR, "reload: could not wait for parent to quit: %s",
+	/* Send quit command to old-main: blocking, wait for receipt.
+	 * The old-main process asks the old-serve processes to quit, however
+	 * if a reload succeeded before, this process is the parent of the
+	 * old-serve processes, so we need to reap the children for it.
+	 */
+	DEBUG(DEBUG_IPC,1, (LOG_INFO, "reload: ipc send quit to main"));
+	cmd = NSD_QUIT_SYNC;
+	if (!write_socket(cmdsocket, &cmd, sizeof(cmd)))
+	{
+		log_msg(LOG_ERR, "problems sending command from reload to oldnsd: %s",
 			strerror(errno));
 	}
-	DEBUG(DEBUG_IPC,1, (LOG_INFO, "reload: ipc reply main %d %d", ret, (int)cmd));
+
+	reload_sync_timeout.tv_sec = RELOAD_SYNC_TIMEOUT;
+	reload_sync_timeout.tv_usec = 0;
+
+	cb_data.base = nsd_child_event_base();
+	cb_data.to_read.cmd = cmd;
+	cb_data.read = 0;
+
+	event_set(&signal_event, SIGCHLD, EV_SIGNAL|EV_PERSIST,
+	    server_reload_handle_sigchld, NULL);
+	if(event_base_set(cb_data.base, &signal_event) != 0
+	|| event_add(&signal_event, NULL) != 0) {
+		log_msg(LOG_ERR, "NSD quit sync: could not add signal event");
+	}
+
+	event_set(&cmd_event, cmdsocket, EV_READ|EV_TIMEOUT|EV_PERSIST,
+	    server_reload_handle_quit_sync_ack, &cb_data);
+	if(event_base_set(cb_data.base, &cmd_event) != 0
+	|| event_add(&cmd_event, &reload_sync_timeout) != 0) {
+		log_msg(LOG_ERR, "NSD quit sync: could not add command event");
+	}
+
+	/* short-lived main loop */
+	event_base_dispatch(cb_data.base);
+
+	/* remove command and signal event handlers */
+	event_del(&cmd_event);
+	event_del(&signal_event);
+	event_base_free(cb_data.base);
+	cmd = cb_data.to_read.cmd;
+
 	if(cmd == NSD_QUIT) {
 		/* small race condition possible here, parent got quit cmd. */
 		send_children_quit(nsd);
 		exit(1);
 	}
-	assert(ret==-1 || ret == 0 || cmd == NSD_RELOAD);
+	assert(cmd == NSD_RELOAD);
 	udb_ptr_unlink(&last_task, nsd->task[nsd->mytask]);
 	task_process_sync(nsd->task[nsd->mytask]);
 #ifdef USE_ZONE_STATS
@@ -2685,7 +2781,7 @@ server_main(struct nsd *nsd)
 				break;
 
 			/* timeout to collect processes. In case no sigchild happens. */
-			timeout_spec.tv_sec = 60;
+			timeout_spec.tv_sec = 1;
 			timeout_spec.tv_nsec = 0;
 
 			/* listen on ports, timeout for collecting terminated children */
diff --git a/tpkg/defunct.tdir/defunct.control.key b/tpkg/defunct.tdir/defunct.control.key
@@ -0,0 +1,40 @@
+-----BEGIN PRIVATE KEY-----
+MIIG/gIBADANBgkqhkiG9w0BAQEFAASCBugwggbkAgEAAoIBgQC9u0DBJJwVyErW
+pRAYQU1kmKa59gQAkmg0mrNUMDa+JYJj5U//XoGODBWJwkhdyO65NwNSVI5bkRzi
+vKcd7kIl8uwD5lTQYl4ArGKX8A1K0JtTXZ8L8+nQJRWexlVLEwjMbwT6MWHLt92A
+8qLKZQxzOf7WN2lj6AuHBC2hSewZuo9rhW57y8uAOiH8OTygFh4inSeozdsCiEq5
+hI4kWpcHHj0KE2Sf2Ngje9U9hDXgcUwRABwIGfT1KG/gT6BQxvrVtaAPQBJxdJHX
+RkM9l857x64LbFHHcM2G3GEHBlcLv+LlA+RKg9zMbppnaZzlfDK9zZx79qPTiPfv
+kH2HEtiPiiulzzDbncXHHJguXE9WIBCpNedXeMffevUqiDb9rnsQ7r9sNtXu9FDH
+qSMbSBX00ZJIEFaauAL1EyFyK6owqLFwbnWI9WlxW+C/2XzDhs9FUH1SRigT8JLY
+wgwwv+uMTTb1x11qyihBrYJZAWXix4oqL9HVUaGetTFAP3LeLYMCAwEAAQKCAYA8
+SH3XoGEzjpMwxftqKX2Eo/f9iAxXfi074Zdu/W63VcWrJdYYJpf1YSOKxL8nhUtM
+cA1PIGym7WGHLTfQC77aved9ibaTC14gyqT875nZlpRzlYCOVK6Kou3Op+e71c1q
+3y9uqzgOeVoYuX893YNqLRgqSirjQnw/JvW0lH8+4YbmO9wR23vKPy5DLJB1sPFc
+QCC0wst+b32GmnZrWJk5+sKcK/NB0q1CAPuM8zAHIxhvImDFtjGdn36Ef+hJIh9O
+LXabPi/ujZ6FOilgl+kYzxcPvBQCxpnet4lGMUd4kXGJzlfkmX7BKQP+nnEHLIl6
+DZOxFPI8yvfJ2sFpBbtwICuJd+EgT7s9pXtI5jIO2qGj/q+ilpGcUemUc9DdVu2d
+KEs7KFLoOTjmxwQq61rrchhpnreNBh4XUvJioHVYk/+ru8vudmb+wlkU7nE0OdPX
+NjHgj6n4ATK4MI7gPrknisOYqQ7j6nkFeyNoDs7DRblP28a5SfTwb4tBYPNQW6kC
+gcEA8BfQL+MRPYb3njPXHXRk7xd10i1EdRDXmD2Q9F3XuHIpXrtUjsgplshovmmo
+9bprmDUJCGFp8RBuEHsktd+H623VFd8nfrvUUcFQBqLXtPmZr32kRwr9SOCkUETn
+LpM9uArzBiTWXmk9RzVuIchhcY5YL1etIrQlVcDwOJ6mOLled/iW5IQ30KGCY6nn
+auwk26ExG0ZG0+LFYoaE9C2AyqlEHW/BRBXB+Uw6NJXuka5z5c4KETcofYcvZHnh
+SIaFAoHBAMpNQ4OwUdSqrCZNgz3QD+eKGONbprePXv5aEHXdYN/y+wyMek4wIhHW
+3UpOKPQT2GaIY0ZuX3JClbzGssadDQZ34laa6s9ExMbrRWvT7+Cg5UU54hjlobyC
+6Lq5RQy9QHgILAEd3H7AiqsgNMKftWmkOqc+xrUVfnkPtZnBC2GioWXcH7MUkIL/
+JMuy/krofIsGokbKcXbMJ85wdBwCgnsSxBK4BpsphrPtQ4eRrafdHxlzK1jwj02n
+YN5D0hA2ZwKBwQDbchiSPGvcnckj62QaAvHiasIA+ekRSvSTrOoZwNBUQNw5PuXZ
+n3AL5l9+ctrAxsRjGU6JP2hysr63B6cVKRt1fyz9Ubsqpgk5NIStnBemzWDAOwl0
++cFp5BGrYF8hfekyl6drHS5mmrAsrtRbwmf2P9Gwa+XOy3M4fyHRzohFXoDPDQL2
+GTde1NB1wazIWNCTgjB9Km3AawxYsnVBLXoDQlUExsmSLoTeFOM6fzPQKfC6wekH
+tS+oFHCn2JKmFHUCgcEAq3rSv821j48cXVtW5A55bMqiThZiOzZMT04IpeABPdO4
++eoZwi0KtBRtf47BKCOqSz9Q52KnhTLruhizcaeuYs08Di1C3G5uXi8PqVb8/mtj
+7EokXPZJIoHUxHJHFYVWIGjM5CVrwgVzhZrHHjJClEmhbiLITLbrfbLTPIFmJFy2
+cqEkPagw1tlvVztiUhy/dRkqdNclVo23MEYiJZv6/lfp2/QhpWSNyfZUR5Y0nRXS
+RsBY/grrbedzNPRIG3YBAoHAYulVIX0Bt5HFJznpoZsAPt5z3LuX1I2qnyFwQOPE
+k9QnSKKj7/BuG5qXWKEur3WgsCWnLMkieSZhfqK3+kBii2gyWgjR1uVsYT/SJuMh
+tqDJOwew2RN3B+xCydIT0PRubeKns1A4qeuAv1Juxw0S9wg6hBGb6GO12SkncYk3
+uQ3Qyahhyo+OIBf5y7BOcLKRm07AI2x6C3MTYyTwAV9VS2ByNsGxtVQysooeaam/
+Lfohlvr92Vv2N1gGcBSyrwTY
+-----END PRIVATE KEY-----
diff --git a/tpkg/defunct.tdir/defunct.control.pem b/tpkg/defunct.tdir/defunct.control.pem
@@ -0,0 +1,24 @@
+-----BEGIN CERTIFICATE-----
+MIIEGjCCAoKgAwIBAgIUdd5j+EPMyI74clpGCDGKjZLy7iMwDQYJKoZIhvcNAQEL
+BQAwDjEMMAoGA1UEAwwDbnNkMB4XDTIzMTExOTE2MjUxNFoXDTQzMDgwNjE2MjUx
+NFowFjEUMBIGA1UEAwwLbnNkLWNvbnRyb2wwggGiMA0GCSqGSIb3DQEBAQUAA4IB
+jwAwggGKAoIBgQC9u0DBJJwVyErWpRAYQU1kmKa59gQAkmg0mrNUMDa+JYJj5U//
+XoGODBWJwkhdyO65NwNSVI5bkRzivKcd7kIl8uwD5lTQYl4ArGKX8A1K0JtTXZ8L
+8+nQJRWexlVLEwjMbwT6MWHLt92A8qLKZQxzOf7WN2lj6AuHBC2hSewZuo9rhW57
+y8uAOiH8OTygFh4inSeozdsCiEq5hI4kWpcHHj0KE2Sf2Ngje9U9hDXgcUwRABwI
+GfT1KG/gT6BQxvrVtaAPQBJxdJHXRkM9l857x64LbFHHcM2G3GEHBlcLv+LlA+RK
+g9zMbppnaZzlfDK9zZx79qPTiPfvkH2HEtiPiiulzzDbncXHHJguXE9WIBCpNedX
+eMffevUqiDb9rnsQ7r9sNtXu9FDHqSMbSBX00ZJIEFaauAL1EyFyK6owqLFwbnWI
+9WlxW+C/2XzDhs9FUH1SRigT8JLYwgwwv+uMTTb1x11qyihBrYJZAWXix4oqL9HV
+UaGetTFAP3LeLYMCAwEAAaNoMGYwDAYDVR0TAQH/BAIwADAWBgNVHREEDzANggtu
+c2QtY29udHJvbDAdBgNVHQ4EFgQU0SC9KMxX0jihcCV1Clj2B7cG7P8wHwYDVR0j
+BBgwFoAU9v/9HVF57j/JYTCObg0ODJkWxW4wDQYJKoZIhvcNAQELBQADggGBAEgk
+p/GZU39ASEupbyo9G6GWYzp4FZQMrfTZsqGthg0jRad0ZymAcBEyFA1kyj2yI2N3
+9pJQ/tq0rt7eggPjSEn05hyWnRMWvsmztpsH4Bpj4wDkUUGvDB4jyPgEJVJQB0+U
+IlhJx0FvfZu6Y2rl1bTwUUlxoaYX4Db297JzoM6nyAEtOeS7PiWpXL86DMCom/8I
+jIRjagRVBtDioZ71OBXvraCy9juYnaUPzynJ8C4ETs5xcdDcMeKX1mUrKryoYAeS
+T61yb1//B7FOMvWo5UKW2u2X/jJIFR1OtawXMwhGUm1TrUhbdD0Sr+dH5hgNx4Ad
+j8idIAgfHOrrPVGMTVactX5b4Lv8m9H6xD7wQVw8NcO2O4aXHNxOmBl9Dex2kaOK
+OntvaBTmbZJ42ydenfnR5U/FrK3u8cnk1oT8qfX31fHS/T4MPIRqvbUZcftGQ4Xn
+xA3eqOFG7idiGENNWln8PbmxBootacpBNCf4Za1Tvr9u8eENAbcBv8mtGp45iQ==
+-----END CERTIFICATE-----
diff --git a/tpkg/defunct.tdir/defunct.dsc b/tpkg/defunct.tdir/defunct.dsc
@@ -0,0 +1,16 @@
+BaseName: defunct
+Version: 1.0
+Description: Test different ways zombie processes can occur
+CreationDate: wo 17 jan 2024 13:44:52 CET
+Maintainer: Willem Toorop
+Category:
+Component:
+Depends:
+Help:
+Pre: defunct.pre
+Post: defunct.post
+Test: defunct.test
+AuxFiles: defunct.nsd.conf.in, defunct.server.key, defunct.server.pem,
+	defunct.control.key, defunct.control.pem,
+Passed:
+Failure:
diff --git a/tpkg/defunct.tdir/defunct.nsd.conf.in b/tpkg/defunct.tdir/defunct.nsd.conf.in
@@ -0,0 +1,21 @@
+server:
+	server-count: 5
+	zonesdir: "."
+	username: ""
+	database: ""
+	verbosity: 2
+	ip-address: 127.0.0.1
+	port: @NSD_PORT@
+	pidfile: "nsd.pid"
+	logfile: "nsd.log"
+	xfrdfile: "xfrd.state"
+	zonelistfile: "zone.list"
+	cookie-secret-file: "nsd-cookiesecrets.txt"
+
+remote-control:
+	control-enable: yes
+	control-port: @NSD_CTRL_PORT@
+	server-key-file: "defunct.server.key"
+	server-cert-file: "defunct.server.pem"
+	control-key-file: "defunct.control.key"
+	control-cert-file: "defunct.control.pem"
diff --git a/tpkg/defunct.tdir/defunct.post b/tpkg/defunct.tdir/defunct.post
@@ -0,0 +1,10 @@
+# #-- defunct.post --#
+# source the master var file when it's there
+[ -f ../.tpkg.var.master ] && source ../.tpkg.var.master
+# source the test var file when it's there
+[ -f .tpkg.var.test ] && source .tpkg.var.test
+
+. ../common.sh
+
+# do your teardown here
+kill_from_pidfile nsd.pid
diff --git a/tpkg/defunct.tdir/defunct.pre b/tpkg/defunct.tdir/defunct.pre
@@ -0,0 +1,27 @@
+# #-- defunct.pre--#
+# source the master var file when it's there
+[ -f ../.tpkg.var.master ] && source ../.tpkg.var.master
+# use .tpkg.var.test for in test variable passing
+[ -f .tpkg.var.test ] && source .tpkg.var.test
+. ../common.sh
+
+# start NSD
+if [ -z "$NSD_PORT" ]; then
+	get_random_port 2
+	NSD_PORT=$RND_PORT
+	NSD_CTRL_PORT=`expr $RND_PORT + 1`
+fi
+
+PRE="../.."
+TPKG_NSD="$PRE/nsd"
+
+# share the vars
+echo "export NSD_PORT=$NSD_PORT" >> .tpkg.var.test
+echo "export NSD_CTRL_PORT=$NSD_CTRL_PORT" >> .tpkg.var.test
+
+sed -e "s/@NSD_PORT@/${NSD_PORT}/g" \
+    -e "s/@NSD_CTRL_PORT@/${NSD_CTRL_PORT}/g" \
+	    defunct.nsd.conf.in > nsd.conf
+
+$TPKG_NSD -c nsd.conf
+wait_nsd_up nsd.log
diff --git a/tpkg/defunct.tdir/defunct.server.key b/tpkg/defunct.tdir/defunct.server.key
@@ -0,0 +1,40 @@
+-----BEGIN PRIVATE KEY-----
+MIIG/QIBADANBgkqhkiG9w0BAQEFAASCBucwggbjAgEAAoIBgQCjsBHexLKQb89F
+S+bj3t+QNWMvWgxWReBj7yCjDXGsMU/SaanMUs/4BgTE7IVeaqQQlYlr4TT0m5HS
+8UWSIp9BSYH+NA6epbCa7iBFKLOsTOvDAsQuWV3aB1ApqQYGp6Ou2TqIF0DupE0x
+jDE32+GEj8p/+jjKhC+BD9XtNf2RyOBQhCgTTlD8oKoPxo+V3T0N8fRymojXElTQ
+SP8cgZdHEUEOUlyiG3ACgv0/wJbpxyvs12nIvLPmWQ4L/QvqROwUCd4uMUmeAPSz
+ctD6kG+sf2lVPg0SOW9Of58DkaWHep6VeEoRoi0SQpJH+mg/EI6NsDLlvyjStkYk
+QuKh26I//QNLetPmTMlCeevMoVPB17cr3AhwudKaKD7EwVobgnuj9cmunUDJG/Mn
+jadbq//ZIQuSdha3ppx3A48CiX1+Iyi7PCv7r3rtQBkcmA1mIWir3zLHujZJ0wdB
+QL636Xz9enVyrCu4D26eJLgH1Vbhwgra2UPdkL9phA70PrL5qD8CAwEAAQKCAYAH
+3rxNOKnZNfcCPu7//8RB+78kFLFJKJzsUSABlvDjNY+/7yKh1012gFuUWEblTj1L
+KtVroW1ZYW12dfFBYHbFvdIJNdmKGS9Pk6lridRxsVUYkDZ8gAF6LSXWivDmOwMs
+sfcVegC+OJoZisMHC4RhSkSTOKWoXwilrOw9Nn5VlxHp7SC8JOTR8KucjJB0xSld
+VUfKhtGTfackSI4zgauTWCoHlUAs7yLIjCG/AfR/kvryrTzB0QjZR2jcfLMo9V1M
+tztJLE/xNEZR4T6QOVT+H7yOQFFrL5Pbsh0FOnjTlPjLhrVm+pY8KuQ8ctmFDk3b
+OI6dUXsRJQpKUUoGg/CCqUETl/aLQ1YJMzZFZ6NhJ1eLLCol2zduz273JPWa48F0
++gs8KXWzMsIRDsmEAVwHURyPUFn3/pxuyIuBB4eqKrzak07bN4oS/OLUZwPN/PXs
+rwzqRgqSBTNo8i4DhWn6OAn+VY2v54VaImg7Y3lBNoc90x0LnnpVcILtkcwAhtkC
+gcEA16qiAgLx3Vt1CqqL4WoFcg4/ZoMCu4dzuExrXzBvy70o8+16WQAlgiYt8e4C
+cBnteblRd5QSghlIC0vksuMycGTCWLnt3Ie0xssrRncPJcQW1/apUZnaMUXQCh4u
+bdx9LcP/onQWVNOgiTzQPG6DpcaQtQIb52VVVCROdv1m66LqWJLiXE+4pECr7HHC
+1u0DHtSVQlOZrKS2HnRsb0cxUvRG0E6jimORkWug+v3eElmK0E9/Icsd3oQLV4s0
+kOBVAoHBAMJM368ZDCdSQx6z7EKsnM5o0lo6DTj10WYbGGXIJELyUZl1SZzZLUe2
+UQ279/ljGN7MFA42GuUeYQYRsX+rQLjIxxg0LDbzhv5iWCgXhKWa10nDoRcNV4tX
+xGUDI+QiROwJ9mPZEgcNpJvPINPGTCFjGhGecooDVc8xSQdxDwqMBbucQOMEqSNN
+wrbyMm3+s5L5bJX6aEmuPcm07SjAZYgTSSu0YU56HwxBTnmmwSSR0QNHUkhIHODO
+bOXX2D0qQwKBwAha8PnY+VvVDjzCjLbH77wSdzxWeQWg4sfSFhrS+pHPG0YP/jxy
+eAGJzx/fU7LaHkwn9NMIJotpxVPCtfKEQND/5lOfqGpxy3wkJOIO/MQ+y4/EW5zw
+s/ZSuRObRbs13pemT0IQ80iIBmEqvPqKYYlLnxGUKlaVo2+mFHAhsxoP7rpNvn1L
+DWE3rDF12tGOzx8jp0CxknOQ0gluA0FwZaXMFu49DbLv1pTmsY7nxPG7DFizq744
+a0jSnsned8PhHQKBwQC3M0aBUWudM7onm9v7bceX80vLWQsYluaukIOrjE9sdBAu
+UmFrK3a/NA8ulPoVwbM0d1W9pJQtTjyLPWmQCAfJcz4N+ztpQKsa9d1LSAmBIAM4
+AOXF14Wg08tQtupDc3Bq4mXX9VdDIUEAzAaWx1UG8esli7FRoh1pc335L0fQWQAa
+yhSAOOfFas9aMfkMr2ff/5+7tdRZDJojsp0enHHfINgQBk7tcMYPhxAYMfvQoFBj
+9/6EOsJH0TrqaF7yJQMCgcATUPBv4R50vvD0AafWch/vJnt2EfbsH89aRmsM8quH
+9+rMcVUqN0yy5YveRsYxDvIpGPMbvXijG2O07xnPX/5yDV2JDGbWEwg9YUS2mb3g
+iGRc5KGJRZnFNdXnAhk8853k6FQbhLlB0fqbYH8x8KtwH9uMn9UkAXFdvr0JKrg7
+9xp+kDb98VxJtXSMeGB5iIgoi5jnAannMiNszk07IoClUpU5f1QZzmz5+UYTNEt9
+PPG/m1FTeHXH407sfqB8frY=
+-----END PRIVATE KEY-----
diff --git a/tpkg/defunct.tdir/defunct.server.pem b/tpkg/defunct.tdir/defunct.server.pem
diff --git a/tpkg/defunct.tdir/defunct.test b/tpkg/defunct.tdir/defunct.test
