Hello.
After executing the function rc4(), the buffer contains garbage. The algorithm itself also works incorrectly.
#include <stdio.h>
#include <stdlib.h>
#define swap(type,a,b,c) do{type t=a[b];a[b]=a[c];a[c]=t;}while(0)
unsigned char *rc4(unsigned char *key, unsigned key_size, unsigned char *buf_in, unsigned buf_in_size, unsigned char *buf_out)
{
unsigned char s[256];
unsigned i, j, c;
for (i = 0; i < 256; s[i] = i++);
for (i = j = 0; i < 256; ++i) {
j = (j + s[i] + key[i % key_size]) & 0xFF;
swap(unsigned char, s, i, j);
}
i = j = c = 0;
while (buf_in_size--) {
i = (i + 1) & 0xFF;
j = (j + s[i]) & 0xFF;
swap(unsigned char, s, i, j);
buf_out[c] = buf_in[c] ^ s[(s[i] + s[j]) & 0xFF];
c++;
}
return buf_out;
}
int entry(void)
{
unsigned char buf[] = "qwerty";
unsigned char key[] = "key";
rc4(key, 3, buf, 6, buf);
for (int i = 0; i < 6; ++i)
{
printf("%02X ", buf[i]);
}
exit(0);
}