i've implemented netty, and i want to implement RSA with it, however it doesn't work. it just throws a: Encrypted size mismatch.
i've literally tried everything.
here's my RS2LoginProtocolDecoder.java:
Code:
package server.net;
import java.math.BigInteger;
import java.security.SecureRandom;
import org.jboss.netty.buffer.ChannelBuffer;
import org.jboss.netty.buffer.ChannelBuffers;
import org.jboss.netty.channel.Channel;
import org.jboss.netty.channel.ChannelFuture;
import org.jboss.netty.channel.ChannelFutureListener;
import org.jboss.netty.channel.ChannelHandlerContext;
import org.jboss.netty.handler.codec.frame.FrameDecoder;
import server.Config;
import server.Connection;
import server.Server;
import server.engine.HandleConnection;
import server.model.players.Client;
import server.model.players.PlayerHandler;
import server.model.players.PlayerSave;
import server.net.PacketBuilder;
import server.util.ISAACCipher;
import server.util.Misc;
public class RS2LoginProtocolDecoder extends FrameDecoder {
private static final BigInteger RSA_MODULUS = new BigInteger("134879924117018689760458617113763010794928714932318058467109552420048747861644217016255598201828589101722456409486834737042810145044971733480806210971246858756450230199459844416953238964275407953368738189689602122143864920074673229804529378820732033043260467820015292026023179060008040229774580922064833792491");
private static final BigInteger RSA_EXPONENT = new BigInteger("90234148541839990140799054102809488461370745952554327409432095095970479290262432684753810024275945810395950055961377271021025802208702575380814921553066912655199489970700117073408649367015293924129928310987520073084130900423361880923927472681140195478097378187149074976649709577953631380990519168830771137473");
private static final int CONNECTED = 0;
private static final int LOGGING_IN = 1;
private int state = CONNECTED;
@Override
protected Object decode(ChannelHandlerContext ctx, Channel channel, ChannelBuffer buffer) throws Exception {
if(!channel.isConnected()) {
return null;
}
switch (state) {
case CONNECTED:
if (buffer.readableBytes() < 2)
return null;
int request = buffer.readUnsignedByte();
if (request != 14) {
System.out.println("Invalid login request: " + request);
channel.close();
return null;
}
buffer.readUnsignedByte();
channel.write(new PacketBuilder().putLong(0).put((byte) 0).putLong(new SecureRandom().nextLong()).toPacket());
state = LOGGING_IN;
return null;
case LOGGING_IN:
if (buffer.readableBytes() < 2) {
return null;
}
int loginType = buffer.readByte();
if (loginType != 16 && loginType != 18) {
System.out.println("Invalid login type: " + loginType);
}
int blockLength = buffer.readByte() & 0xff;
if (buffer.readableBytes() < blockLength) {
return null;
}
buffer.readByte();
for (int i = 0; i < 9; i++)
buffer.readInt();
int loginEncryptPacketSize = -1;
loginEncryptPacketSize--;
if(loginEncryptPacketSize != (buffer.readLong() & 0xff)) {
System.out.println("Encrypted size mismatch.");
channel.close();
return false;
}
byte[] encryptionBytes = new byte[loginEncryptPacketSize];
buffer.writeBytes(encryptionBytes);
ChannelBuffer rsaBuffer = ChannelBuffers.wrappedBuffer(new BigInteger(encryptionBytes).modPow(RSA_EXPONENT, RSA_MODULUS).toByteArray());
if((rsaBuffer.readLong() & 0xff) != 10) {
System.out.println("Encrypted id != 10.");
channel.close();
return false;
}
int uid = rsaBuffer.readInt();
if(uid == 0 || uid == 99735086) {
channel.close();
return false;
}
final long clientHalf = rsaBuffer.readLong();
final long serverHalf = rsaBuffer.readLong();
final int[] isaacSeed = { (int) (clientHalf >> 32), (int) clientHalf, (int) (serverHalf >> 32), (int) serverHalf };
final ISAACCipher inCipher = new ISAACCipher(isaacSeed);
for (int i = 0; i < isaacSeed.length; i++)
isaacSeed[i] += 50;
final ISAACCipher outCipher = new ISAACCipher(isaacSeed);
final int version = buffer.readInt();
final String name = Misc.formatPlayerName(Misc.getRS2String(buffer));
final String pass = Misc.getRS2String(buffer);
channel.getPipeline().replace("decoder", "decoder", new RS2ProtocolDecoder(inCipher));
return login(channel, inCipher, outCipher, version, name, pass);
}
return null;
}
private static Client login(Channel channel, ISAACCipher inCipher, ISAACCipher outCipher, int version, String name, String pass) {
int returnCode = 2;
if (!name.matches("[A-Za-z0-9 ]+")) {
returnCode = 4;
}
if (name.length() > 12) {
returnCode = 8;
}
Client cl = new Client(channel, -1);
cl.playerName = name;
cl.playerName2 = cl.playerName;
cl.playerPass = pass;
cl.outStream.packetEncryption = outCipher;
cl.saveCharacter = false;
cl.isActive = true;
if (Connection.isNamedBanned(cl.playerName)) {
returnCode = 4;
}
if (PlayerHandler.isPlayerOn(name)) {
returnCode = 5;
}
if (PlayerHandler.getPlayerCount() >= Config.MAX_PLAYERS) {
returnCode = 7;
}
if (HandleConnection.UpdateServer) {
returnCode = 14;
}
if (returnCode == 2) {
int load = PlayerSave.loadGame(cl, cl.playerName, cl.playerPass);
if (load == 0)
cl.addStarter = true;
if (load == 3) {
returnCode = 3;
cl.saveFile = false;
} else {
for (int i = 0; i < cl.playerEquipment.length; i++) {
if (cl.playerEquipment[i] == 0) {
cl.playerEquipment[i] = -1;
cl.playerEquipmentN[i] = 0;
}
}
if (!Server.playerHandler.newPlayerClient(cl)) {
returnCode = 7;
cl.saveFile = false;
} else {
cl.saveFile = true;
}
}
}
if(returnCode == 2) {
cl.saveCharacter = true;
cl.packetType = -1;
cl.packetSize = 0;
final PacketBuilder bldr = new PacketBuilder();
bldr.put((byte) 2);
if (cl.playerRights == 3) {
bldr.put((byte) 2);
} else {
bldr.put((byte) cl.playerRights);
}
bldr.put((byte) 0);
channel.write(bldr.toPacket());
} else {
System.out.println("returncode:" + returnCode);
sendReturnCode(channel, returnCode);
return null;
}
synchronized (PlayerHandler.lock) {
cl.initialize();
cl.initialized = true;
}
return cl;
}
public static void sendReturnCode(final Channel channel, final int code) {
channel.write(new PacketBuilder().put((byte) code).toPacket()).addListener(new ChannelFutureListener() {
@Override
public void operationComplete(final ChannelFuture arg0) throws Exception {
arg0.getChannel().close();
}
});
}
}
and here's my doKeys() void, Stream.java:
Code:
public void doKeys() {
int i = currentOffset;
currentOffset = 0;
byte abyte0[] = new byte[i];
readBytes(i, 0, abyte0);
BigInteger biginteger2 = new BigInteger(abyte0);
BigInteger biginteger3 = biginteger2.modPow(RSA_EXPONENT, RSA_MODULUS);
byte abyte1[] = biginteger3.toByteArray();
currentOffset = 0;
writeWordBigEndian(abyte1.length);
writeBytes(abyte1, abyte1.length, 0);
}
any help is much appreciated, thank you!