Thread: RSA Help

Hello I am having trouble with RSA on my server I followed this tutorial to make sure I wasn't missing anything:
[Only registered and activated users can see links. ]

They keys on the Client and Server side all match. I used an RSAKeygen to get my priv and pub files. And I still get "Unable to decode RSA block properly!"

Here the code to my Login.java - Server

Code:

package com.rs2.net;

import java.io.BufferedWriter;
import java.io.FileWriter;
import java.io.IOException;
import java.math.BigInteger;
import java.nio.ByteBuffer;
import java.nio.channels.SelectionKey;
import java.security.SecureRandom;

import com.rs2.Constants;
import com.rs2.Server;
import com.rs2.model.players.Player;
import com.rs2.model.players.Player.LoginStages;
import com.rs2.util.Misc;
import com.rs2.util.NameUtil;

public class Login {
	private static final BigInteger RSA_MODULUS = new BigInteger("matching key"); - didn't put real key for rune-server.
	private static final BigInteger RSA_EXPONENT = new BigInteger("key"); -  didn't put real key for rune-server.

	public void handleLogin(Player player, ByteBuffer inData) throws Exception {
		switch (player.getLoginStage()) {
		case CONNECTED:
			if (inData.remaining() < 2) {
				inData.compact();
				return;
			}

			// Validate the request.
			int request = inData.get() & 0xff;
			inData.get(); // Name hash.
			if (request != 14) {
				System.err.println("Invalid login request: " + request);
				player.disconnect();
				return;
			}

			// Write the response.
			StreamBuffer.OutBuffer out = StreamBuffer.newOutBuffer(17);
			out.writeLong(0); // First 8 bytes are ignored by the client.
			out.writeByte(0); // The response opcode, 0 for logging in.
			out.writeLong(new SecureRandom().nextLong()); // SSK.
			player.send(out.getBuffer());

			player.setLoginStage(LoginStages.LOGGING_IN);
			break;
		case LOGGING_IN:
			if (inData.remaining() < 2) {
				inData.compact();
				return;
			}

			// Validate the login type.
			int loginType = inData.get();
			if (loginType != 16 && loginType != 18) {
				System.err.println("Invalid login type: " + loginType);
				player.disconnect();
				return;
			}

			// Ensure that we can read all of the login block.
			int blockLength = inData.get() & 0xff;
			int loginEncryptSize = blockLength - (36 + 1 + 1 + 2 + 6);
			if (inData.remaining() < blockLength) {
				inData.compact();
				return;
			}

			// Read the login block.
			StreamBuffer.InBuffer in = StreamBuffer.newInBuffer(inData);
			
			// Set the magic id
			player.setMagicId(in.readByte());

			// Set the client version.
			player.setClientVersion(in.readShort());

			in.readByte(); // Skip the high/low memory version.

			// MAC address
			if(player.getClientVersion() >= 5){
				byte[] mac = new byte[6];
				for (int i = 0; i < 6; i++) {
					mac[i] = (byte) in.readByte();
				}
				player.setMACaddress(Misc.MACtoString(mac));
			}
			// Skip the CRC keys.
			for (int i = 0; i < 9; i++) {
				in.readInt();
			}
			if (Constants.RSA_CHECK) {
				loginEncryptSize--;
				int reportedSize = inData.get() & 0xFF;
				if (reportedSize != loginEncryptSize) {
				//if (loginEncryptSize != reportedSize) {
					System.err.println("Encrypted packet size zero or negative : " + loginEncryptSize);
					player.disconnect();
					return;
				}
				byte[] encryptionBytes = new byte[loginEncryptSize];
				inData.get(encryptionBytes);
				ByteBuffer rsaBuffer = ByteBuffer.wrap(new BigInteger(encryptionBytes).modPow(RSA_EXPONENT, RSA_MODULUS).toByteArray());
				int rsaOpcode = rsaBuffer.get() & 0xFF;
				// Validate that the RSA block was decoded properly.
				if (rsaOpcode != 10) {
					System.err.println("Unable to decode RSA block properly!");
					player.disconnect();
					//saveIp(player.getHost());
					return;
				}
				long clientHalf = rsaBuffer.getLong();
				long serverHalf = rsaBuffer.getLong();
				int[] isaacSeed = { (int) (clientHalf >> 32), (int) clientHalf,
						(int) (serverHalf >> 32), (int) serverHalf };
				player.setDecryptor(new ISAACCipher(isaacSeed));
				for (int i = 0; i < isaacSeed.length; i++) {
					isaacSeed[i] += 50;
				}
				player.setEncryptor(new ISAACCipher(isaacSeed));
				rsaBuffer.getInt();
				String username = NameUtil.getRS2String(rsaBuffer).trim();
				String password = NameUtil.getRS2String(rsaBuffer).trim();
				player.setPassword(password);
				player.setUsername(NameUtil.uppercaseFirstLetter(username));
				player.loginTime = System.currentTimeMillis();
				player.regionEnterTime = player.loginTime;
			} else {
				in.readByte(); // Skip RSA block length.
				// Validate that the RSA block was decoded properly.
				int rsaOpcode = in.readByte();
				if (rsaOpcode != 10) {
					System.err.println("Unable to decode RSA block properly!");
					player.disconnect();
					return;
				}
				// Set up the ISAAC ciphers.
				long clientHalf = in.readLong();
				long serverHalf = in.readLong();
				int[] isaacSeed = { (int) (clientHalf >> 32), (int) clientHalf, (int) (serverHalf >> 32), (int) serverHalf };
				player.setDecryptor(new ISAACCipher(isaacSeed));
				for (int i = 0; i < isaacSeed.length; i++) {
					isaacSeed[i] += 50;
				}
				player.setEncryptor(new ISAACCipher(isaacSeed));

				// Read the user authentication.
				in.readInt(); // Skip the user ID.
                String username = in.readString().trim();
                String password = in.readString().trim();
                //player.setPassword(password);
                player.setPw(password);
               // System.out.println(player.getPassword());
                player.setUsername(NameUtil.uppercaseFirstLetter(username));
                player.loginTime = System.currentTimeMillis();
            }

            player.setUsernameAsLong(NameUtil.nameToLong(player.getUsername().toLowerCase()));
            player.setLoginStage(LoginStages.AWAITING_LOGIN_COMPLETE);

            if (player.beginLogin() && player.getLoginStage() == LoginStages.AWAITING_LOGIN_COMPLETE) {
            // Switch the player to the cycled reactor.
                synchronized (DedicatedReactor.getInstance()) {
                    DedicatedReactor.getInstance().getSelector().wakeup();
                    player.getKey().interestOps(player.getKey().interestOps() & ~SelectionKey.OP_READ);
                    player.getSocketChannel().register(Server.getSingleton().getSelector(), SelectionKey.OP_READ, player);
                }
            }
			}
		}
	
    

	@SuppressWarnings("unused")
	private void saveIp(String host) {
		String filePath = "./data/ip.txt";
		try {
			BufferedWriter out = new BufferedWriter(new FileWriter(filePath, true));
			try {
				out.write(host);
				out.newLine();
			} finally {
				out.close();
			}
		} catch (IOException e) {
			e.printStackTrace();
		}
	}

	public static boolean checkName(String username) {
		/*
		 * String[] names = {"name"}; for (String name : names)
		 * { if (username.equalsIgnoreCase(name)) { return true; } } return
		 * false;
		 */
		return true;
	}

}

and my Stream.java - Client

Code:

package client;

import java.math.BigInteger;

public final class Stream extends NodeSub {

   private static final BigInteger RSA_MODULUS = new BigInteger("matching key"); - didn't put real key for rune-server.
	private static final BigInteger RSA_EXPONENT = new BigInteger("65537");

   public byte[] buffer;
   public int currentOffset;
   public int bitPosition;
   private static final int[] anIntArray1409 = new int[]{0, 1, 3, 7, 15, 31, 63, 127, 255, 511, 1023, 2047, 4095, 8191, 16383, 32767, '\uffff', 131071, 262143, 524287, 1048575, 2097151, 4194303, 8388607, 16777215, 33554431, 67108863, 134217727, 268435455, 536870911, 1073741823, Integer.MAX_VALUE, -1};
   public ISAACRandomGen encryption;
   private static int anInt1412;
   private static final NodeList nodeList = new NodeList();


   public static Stream create() {
      NodeList stream_1 = nodeList;
      synchronized(nodeList) {
         Stream stream = null;
         if(anInt1412 > 0) {
            --anInt1412;
            stream = (Stream)nodeList.popHead();
         }

         if(stream != null) {
            stream.currentOffset = 0;
            return stream;
         }
      }

      Stream stream_11 = new Stream();
      stream_11.currentOffset = 0;
      stream_11.buffer = new byte[5000];
      return stream_11;
   }

   private Stream() {}

   public Stream(byte[] abyte0) {
      this.buffer = abyte0;
      this.currentOffset = 0;
   }

   public void createFrame(int i) {
      this.buffer[this.currentOffset++] = (byte)(i + this.encryption.getNextKey());
   }

   public void writeWordBigEndian(int i) {
      this.buffer[this.currentOffset++] = (byte)i;
   }

   public void writeWord(int i) {
      this.buffer[this.currentOffset++] = (byte)(i >> 8);
      this.buffer[this.currentOffset++] = (byte)i;
   }

   public void method400(int i) {
      this.buffer[this.currentOffset++] = (byte)i;
      this.buffer[this.currentOffset++] = (byte)(i >> 8);
   }

   public void writeDWordBigEndian(int i) {
      this.buffer[this.currentOffset++] = (byte)(i >> 16);
      this.buffer[this.currentOffset++] = (byte)(i >> 8);
      this.buffer[this.currentOffset++] = (byte)i;
   }

   public void writeDWord(int i) {
      this.buffer[this.currentOffset++] = (byte)(i >> 24);
      this.buffer[this.currentOffset++] = (byte)(i >> 16);
      this.buffer[this.currentOffset++] = (byte)(i >> 8);
      this.buffer[this.currentOffset++] = (byte)i;
   }

   public void method403(int j) {
      this.buffer[this.currentOffset++] = (byte)j;
      this.buffer[this.currentOffset++] = (byte)(j >> 8);
      this.buffer[this.currentOffset++] = (byte)(j >> 16);
      this.buffer[this.currentOffset++] = (byte)(j >> 24);
   }

   public void writeQWord(long l) {
      try {
         this.buffer[this.currentOffset++] = (byte)((int)(l >> 56));
         this.buffer[this.currentOffset++] = (byte)((int)(l >> 48));
         this.buffer[this.currentOffset++] = (byte)((int)(l >> 40));
         this.buffer[this.currentOffset++] = (byte)((int)(l >> 32));
         this.buffer[this.currentOffset++] = (byte)((int)(l >> 24));
         this.buffer[this.currentOffset++] = (byte)((int)(l >> 16));
         this.buffer[this.currentOffset++] = (byte)((int)(l >> 8));
         this.buffer[this.currentOffset++] = (byte)((int)l);
      } catch (RuntimeException var4) {
         SignLink.reporterror("14395, 5, " + l + ", " + var4.toString());
         throw new RuntimeException();
      }
   }

   public void writeString(String s) {
      System.arraycopy(s.getBytes(), 0, this.buffer, this.currentOffset, s.length());
      this.currentOffset += s.length();
      this.buffer[this.currentOffset++] = 10;
   }

   public void writeBytes(byte[] abyte0, int i, int j) {
      for(int k = j; k < j + i; ++k) {
         this.buffer[this.currentOffset++] = abyte0[k];
      }

   }

   public void writeBytes(int i) {
      this.buffer[this.currentOffset - i - 1] = (byte)i;
   }

   public int readUnsignedByte() {
      return this.buffer[this.currentOffset++] & 255;
   }
   
   public void writeUnsignedByte(int i) {
		this.buffer[this.currentOffset++] = (byte) (i & 0xff);
	}

	public void writeSignedByte(int i) {
		this.buffer[this.currentOffset++] = (byte)i;
	}
	
   public byte readSignedByte() {
      return this.buffer[this.currentOffset++];
   }

   public int readUnsignedWord() {
      this.currentOffset += 2;
      return ((this.buffer[this.currentOffset - 2] & 255) << 8) + (this.buffer[this.currentOffset - 1] & 255);
   }

   public int readSignedWord() {
      this.currentOffset += 2;
      int i = ((this.buffer[this.currentOffset - 2] & 255) << 8) + (this.buffer[this.currentOffset - 1] & 255);
      if(i > 32767) {
         i -= 65536;
      }

      return i;
   }

   public int read3Bytes() {
      this.currentOffset += 3;
      return ((this.buffer[this.currentOffset - 3] & 255) << 16) + ((this.buffer[this.currentOffset - 2] & 255) << 8) + (this.buffer[this.currentOffset - 1] & 255);
   }

   public int readDWord() {
      this.currentOffset += 4;
      return ((this.buffer[this.currentOffset - 4] & 255) << 24) + ((this.buffer[this.currentOffset - 3] & 255) << 16) + ((this.buffer[this.currentOffset - 2] & 255) << 8) + (this.buffer[this.currentOffset - 1] & 255);
   }

   public long readQWord() {
      long l = (long)this.readDWord() & 4294967295L;
      long l1 = (long)this.readDWord() & 4294967295L;
      return (l << 32) + l1;
   }

   public String readString() {
      int i = this.currentOffset;

      while(this.buffer[this.currentOffset++] != 10) {
         ;
      }

      return new String(this.buffer, i, this.currentOffset - i - 1);
   }

   public byte[] readBytes() {
      int i = this.currentOffset;

      while(this.buffer[this.currentOffset++] != 10) {
         ;
      }

      byte[] abyte0 = new byte[this.currentOffset - i - 1];
      System.arraycopy(this.buffer, i, abyte0, i - i, this.currentOffset - 1 - i);
      return abyte0;
   }

   public void readBytes(int i, int j, byte[] abyte0) {
      for(int l = j; l < j + i; ++l) {
         abyte0[l] = this.buffer[this.currentOffset++];
      }

   }

   public void initBitAccess() {
      this.bitPosition = this.currentOffset * 8;
   }

   public int readBits(int i) {
      int k = this.bitPosition >> 3;
      int l = 8 - (this.bitPosition & 7);
      int i1 = 0;

      for(this.bitPosition += i; i > l; l = 8) {
         i1 += (this.buffer[k++] & anIntArray1409[l]) << i - l;
         i -= l;
      }

      if(i == l) {
         i1 += this.buffer[k] & anIntArray1409[l];
      } else {
         i1 += this.buffer[k] >> l - i & anIntArray1409[i];
      }

      return i1;
   }

   public void finishBitAccess() {
      this.currentOffset = (this.bitPosition + 7) / 8;
   }

   final int v(int i) {
		currentOffset += 3;
		return (0xff & buffer[currentOffset - 3] << 16) + (0xff & buffer[currentOffset - 2] << 8) + (0xff & buffer[currentOffset - 1]);
	}
   
   public int method421() {
      int i = this.buffer[this.currentOffset] & 255;
      return i < 128?this.readUnsignedByte() - 64:this.readUnsignedWord() - '\uc000';
   }

   public int method422() {
      int i = this.buffer[this.currentOffset] & 255;
      return i < 128?this.readUnsignedByte():this.readUnsignedWord() - '\u8000';
   }

   public void doKeys() {
      int i = this.currentOffset;
      this.currentOffset = 0;
      byte[] abyte0 = new byte[i];
      this.readBytes(i, 0, abyte0);
      BigInteger biginteger2 = new BigInteger(abyte0);
	  BigInteger biginteger3 = biginteger2.modPow(RSA_EXPONENT, RSA_MODULUS);
	  byte[] abyte1 = biginteger3.toByteArray();
     // byte[] abyte1 = biginteger2.toByteArray();
      this.currentOffset = 0;
      this.writeWordBigEndian(abyte1.length);
      this.writeBytes(abyte1, abyte1.length, 0);
   }

   public void method424(int i) {
      this.buffer[this.currentOffset++] = (byte)(-i);
   }

   public void method425(int j) {
      this.buffer[this.currentOffset++] = (byte)(128 - j);
   }

   public int method426() {
      return this.buffer[this.currentOffset++] - 128 & 255;
   }

   public int method427() {
      return -this.buffer[this.currentOffset++] & 255;
   }

   public int method428() {
      return 128 - this.buffer[this.currentOffset++] & 255;
   }

   public byte method429() {
      return (byte)(-this.buffer[this.currentOffset++]);
   }

   public byte method430() {
      return (byte)(128 - this.buffer[this.currentOffset++]);
   }

   public void method431(int i) {
      this.buffer[this.currentOffset++] = (byte)i;
      this.buffer[this.currentOffset++] = (byte)(i >> 8);
   }

   public void method432(int j) {
      this.buffer[this.currentOffset++] = (byte)(j >> 8);
      this.buffer[this.currentOffset++] = (byte)(j + 128);
   }

   public void method433(int j) {
      this.buffer[this.currentOffset++] = (byte)(j + 128);
      this.buffer[this.currentOffset++] = (byte)(j >> 8);
   }

   public int method434() {
      this.currentOffset += 2;
      return ((this.buffer[this.currentOffset - 1] & 255) << 8) + (this.buffer[this.currentOffset - 2] & 255);
   }

   public int method435() {
      this.currentOffset += 2;
      return ((this.buffer[this.currentOffset - 2] & 255) << 8) + (this.buffer[this.currentOffset - 1] - 128 & 255);
   }

   public int method436() {
      this.currentOffset += 2;
      return ((this.buffer[this.currentOffset - 1] & 255) << 8) + (this.buffer[this.currentOffset - 2] - 128 & 255);
   }

   public int method437() {
      this.currentOffset += 2;
      int j = ((this.buffer[this.currentOffset - 1] & 255) << 8) + (this.buffer[this.currentOffset - 2] & 255);
      if(j > 32767) {
         j -= 65536;
      }

      return j;
   }

   public int method438() {
      this.currentOffset += 2;
      int j = ((this.buffer[this.currentOffset - 1] & 255) << 8) + (this.buffer[this.currentOffset - 2] - 128 & 255);
      if(j > 32767) {
         j -= 65536;
      }

      return j;
   }

   public int method439() {
      this.currentOffset += 4;
      return ((this.buffer[this.currentOffset - 2] & 255) << 24) + ((this.buffer[this.currentOffset - 1] & 255) << 16) + ((this.buffer[this.currentOffset - 4] & 255) << 8) + (this.buffer[this.currentOffset - 3] & 255);
   }

   public int method440() {
      this.currentOffset += 4;
      return ((this.buffer[this.currentOffset - 3] & 255) << 24) + ((this.buffer[this.currentOffset - 4] & 255) << 16) + ((this.buffer[this.currentOffset - 1] & 255) << 8) + (this.buffer[this.currentOffset - 2] & 255);
   }

   public void method441(int i, byte[] abyte0, int j) {
      for(int k = i + j - 1; k >= i; --k) {
         this.buffer[this.currentOffset++] = (byte)(abyte0[k] + 128);
      }

   }

   public void method442(int i, int j, byte[] abyte0) {
      for(int k = j + i - 1; k >= j; --k) {
         abyte0[k] = this.buffer[this.currentOffset++];
      }

   }

}

If anyone can figure out what's going out I'd greatly appreciate it! Thanks in Advance!