I've done a small kernel hack, and it doesn't work. I suspect the problem is some deep kernel magic ("blame the compiler" :-)), so I'm hoping that posting it here will lead to some kindly kernel hacker spotting the trouble.

The idea is to make an "emergency rescue" feature for a thrasing system. Recently my desktop machine was thrashing so badly that I couldn't rsh into it, couldn't kill the X server with C-M-Backspace, and basically just couldn't get it working again. It did however have perfectly good pings times during the thrasing. I assume that the kernel was healthy but all userspace processes were swapped into oblivion, and I ended up cycling the power to recover.

The attempted solution is a kernel-space program that simply kills the process using the most physical memory. It is implemented as an iptables target, so that it can be triggered remotely by sending a magic packet that matches some special iptables rule. Fun, huh?

Here's the code for the iptables target, which freezes the whole machine when I try to use it. See the problem?

/*-*- linux-c -*-
 *
 * This is a module for recovering a system that is thrashing. When
 * trigged by a matched packet, it will kill the task that is using
 * the most physical memory (RSS). Not too subtle, but hopefully it
 * beats hitting the power button when userspace is totally thrashed
 * out of operation.
 *
 * -- luke@bluetail.com
 *
 */
#include <linux/sched.h>
#include <linux/module.h>
#include <linux/skbuff.h>
#include <linux/ip.h>
#include <linux/spinlock.h>
#include <net/icmp.h>
#include <net/udp.h>
#include <net/tcp.h>
#include <linux/netfilter_ipv4/ip_tables.h>

 struct in_device;
#include <net/route.h>

 static unsigned int
ipt_killfatso_target(struct sk_buff **pskb,
	       unsigned int hooknum,
	       const struct net_device *in,
	       const struct net_device *out,
	       const void *targinfo,
	       void *userinfo)
{
        struct task_struct *p, *fatso = NULL;
        int fatso_rss;
        /* Find the "fatso" process -- the one with the most RSS */
        read_lock(&tasklist_lock);
        for_each_task(p) {
                spin_lock(&p->mm->page_table_lock);
                if (fatso == NULL || (p->mm->rss > fatso_rss)) {
                        fatso = p;
                        fatso_rss = p->mm->rss;
                }
                spin_unlock(&p->mm->page_table_lock);
        }
        /* And kill him.. */
        if (fatso != NULL) {    /* presumably there is some process.. */
                printk(KERN_NOTICE "killing fatso: %d", fatso->pid);
                force_sig(SIGKILL, fatso);
        }
        /* Unlock the task list, *after* sending the signal - this
           seems to be important. */
        read_unlock(&tasklist_lock);
        return IPT_CONTINUE;
}

 static int ipt_killfatso_checkentry(const char *tablename,
			      const struct ipt_entry *e,
			      void *targinfo,
			      unsigned int targinfosize,
			      unsigned int hook_mask)
 {
	return 1;
}

 static struct ipt_target ipt_killfatso_reg
= { { NULL, NULL },
    "KILLFATSO",
    ipt_killfatso_target,
    ipt_killfatso_checkentry,
    NULL,
    THIS_MODULE };

 static int __init init(void)
{
	if (ipt_register_target(&ipt_killfatso_reg)) {
		return -EINVAL;
	}

 	return 0;
}

 static void __exit fini(void)
{
	ipt_unregister_target(&ipt_killfatso_reg);
}

 module_init(init);
module_exit(fini);
/* MODULE_LICENSE("GPL"); */