Login to website programmatically returns login HTML

This site uses Java Servlet authentication mechanism. When you try to access any password protected page, the server internally issues a redirect that returns the content of the login page. After you post a valid username / password, it redirects you to the required URL.

Please note that this is not a redirect, it is managed by the application server and the client does not see what is happening on the server side.

To get this to work, you need to send a GET to the page you are trying to get. The server responds with the content of the login page and the session ID as a cookie. You have to store the session id and submit it when submitting to the form action.

Since I don't have valid credentials to verify, I cannot guarantee that this will work. If the credentials are invalid, the server will issue a series of redirects that will ultimately change the session, probably because it is invalid and recreated.

Please try this with valid credentials and let me know if it works. If there are redirects on successful login, it may be necessary to disable automatic redirects and do them programmatically, checking for session ID changes. Depending on the application server, it can change the session ID for security purposes.

However, this MAY work when you submit valid credentials.

import java.io.BufferedReader;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLEncoder;
import java.util.List;
import java.util.ListIterator;
import java.util.Map;

public class Scraper {

    private static final String USERNAME = "user@test.com";
    private static final String PASSWORD = "secret";
    private static final String GET_URL  = "https://parents.mtsd.k12.nj.us/genesis/parents?tab1=studentdata&tab2=gradebook&tab3=weeklysummary&studentid=100916&action=form";
    private static final String POST_URL = "https://parents.mtsd.k12.nj.us/genesis/j_security_check";

    public static void main(String[] args) {
        String cookies = doGet(GET_URL);
        doPost(POST_URL, cookies);
    };

    /**
     * Send the initial GET request which will forward to the login page
     * and retrieve cookies sent by the server.
     * Cookies are formatted according to HTTP specification so they can be
     * passed to the next request Cookie header.
     * @param getURL URL to get
     */
    public static String doGet(String getURL) {
        StringBuilder formattedCookies = new StringBuilder();
        try {
            URL url = new URL(getURL);
            HttpURLConnection conn = (HttpURLConnection) url.openConnection();
            InputStream is = conn.getInputStream();
            Map<String, List<String>> headers = conn.getHeaderFields();
            List<String> cookies = headers.get("Set-Cookie");
            ListIterator<String> it = cookies.listIterator();
            while (it.hasNext()) {
                String[] parts = it.next().split("; ");
                formattedCookies.append(parts[0]);
                if (it.hasNext()) {
                    formattedCookies.append("; ");
                }
            }
            System.out.println("\n\nGET OUTPUT");
            printContent(is);
        } catch (Exception e) {
            System.out.println(e.getLocalizedMessage());
        }
        return formattedCookies.toString();
    }

    /**
     * Post the form parameters and get page content.
     * @param postURL URL to post to
     * @param cookies The cookies to send
     */
    public static void doPost(String postURL, String cookies) {
        try {
            String postData = String.format("j_username=%s&j_password=%s",
                URLEncoder.encode(USERNAME, "UTF-8"), PASSWORD);
            URL url = new URL(postURL);
            HttpURLConnection conn = (HttpURLConnection) url.openConnection();
            conn.setRequestMethod("POST");
            conn.setDoOutput(true);
            conn.setInstanceFollowRedirects(true);
            conn.setRequestProperty("Cookie", cookies);
            OutputStreamWriter out = new OutputStreamWriter(conn.getOutputStream());
            out.write(postData);
            out.close();
            InputStream is = conn.getInputStream();
            System.out.println("\n\nPOST OUTPUT");
            printContent(is);
        } catch (Exception e) {
            System.out.println(e.getLocalizedMessage());
        }
    }    

    public static void printContent(InputStream is) throws IOException {
        BufferedReader reader = new BufferedReader(new InputStreamReader(is));
        String line;
        while ((line = reader.readLine()) != null) {
            System.out.println(line);
        }
    }

}

      

        
        
        
      

    

You are trying to get the Set-Cookie header from the connection, which, if you notice in the headers, has two entries. One with the last visited entry and the other with the JSessionID entry .

In your search for cookies, the JSessionID entry is being ignored. Extract the entries as a list and set it correctly.

    List<String> cookies = connection.getHeaderFields().get("Set-Cookie");
    for (String cookie : cookies) {
        connection.addRequestProperty("Cookie", cookie.split(";", 2)[0]);
    }

      

        
        
        
      

    

Edit: Try the below code. This gives me an unknown username and password.

import java.io.DataOutputStream;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.List;

import org.apache.commons.io.IOUtils;

public class Scraper {

static String formData = "j_username=user&j_password=pass";
static String link = "https://parents.mtsd.k12.nj.us/genesis/parents?tab1=studentdata&tab2=gradebook&tab3=weeklysummary&studentid=100916&action=form";

public static void main(String[] args) {

    String display = postData(link, formData);
    System.out.print(display);

}

public static String postData(String url, String data) {

    URL link = null;
    HttpURLConnection connection = null;
    StringBuffer stringBuffer = new StringBuffer();
    DataOutputStream dataOutputStream = null;
    String document = null;

    try {
        link = new URL(url);
    } catch (Exception e) {
        System.out.print(e);
    }

    try {
        connection = (HttpURLConnection) link.openConnection();
        connection.setRequestMethod("GET");
    } catch (Exception e) {
        System.out.print(e);
    }

    connection.setRequestProperty("Accept",
            "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
    connection.setRequestProperty("User-Agent",
            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36");
    connection.setRequestProperty("Accept-Encoding", "gzip, deflate, br");
    connection.setRequestProperty("Accept-Language", "en-US,en;q=0.8");
    // connection.setRequestProperty("Cookie","JSESSIONID=82714DD38C05837934F515C5E0BFD1E3;
    // lastvisit=3626F6FB689948B09CEF3DDA56116B7C");
    connection.setDoInput(true);
    connection.setDoOutput(true);
    // connection.setInstanceFollowRedirects(true);

    // setCookie(connection);

    // String cookie = connection.getHeaderField("Set-Cookie"); //get
    // cookies for session
    List<String> cookies = connection.getHeaderFields().get("Set-Cookie");

    // try {
    // connection.connect();
    // }catch (Exception e){System.out.print(e);}

    // try {
    // dataOutputStream = new
    // DataOutputStream(connection.getOutputStream());
    // }catch (Exception e){System.out.print(e);}

    // try {
    // IOUtils.write(stringBuffer.toString(),dataOutputStream,"UTF-8");
    // }catch (Exception e){System.out.print(e);}

    // handle redirects
    try {

        // post data
        String postString = data;
        stringBuffer.append(postString);

        URL redURL = null;

        try {
            redURL = new URL("https://parents.mtsd.k12.nj.us/genesis/j_security_check");
        } catch (Exception e) {
            System.out.print(e);
        }

        connection = (HttpURLConnection) redURL.openConnection();
        connection.setRequestMethod("POST");
        connection.setRequestProperty("Accept",
                "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
        connection.setRequestProperty("User-Agent",
                "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36");
        connection.setRequestProperty("Accept-Encoding", "gzip, deflate, br");
        connection.setRequestProperty("Accept-Language", "en-US,en;q=0.8");
        // connection.setRequestProperty("Cookie",cookie);
        for (String cookie : cookies) {
            connection.addRequestProperty("Cookie", cookie.split(";", 2)[0]);
        }
        connection.setDoInput(true);
        connection.setDoOutput(true);
        connection.setInstanceFollowRedirects(true);

        // connection.connect();

    } catch (Exception e) {
        System.out.print(e);
    }

    InputStream inputStream = null;

    String cookie2 = null;

    try {
        dataOutputStream = new DataOutputStream(connection.getOutputStream());
    } catch (Exception e) {
        System.out.print(e);
    }

    try {
        IOUtils.write(stringBuffer.toString(), dataOutputStream, "UTF-8");

        System.out.println(stringBuffer.toString());

        cookie2 = connection.getHeaderField("Set-Cookie"); // get cookies
                                                            // for session

    } catch (Exception e) {
        System.out.print(e);
    }

    URL fLink = null;

    try {
        fLink = new URL(
                "https://parents.mtsd.k12.nj.us/genesis/parents?tab1=studentdata&tab2=gradebook&tab3=weeklysummary&studentid=100916&action=form"
                        + stringBuffer.toString());
    } catch (Exception e) {
        System.out.print(e);
    }

    HttpURLConnection conn = null;

    try {
        conn = (HttpURLConnection) fLink.openConnection();
        conn.setRequestMethod("GET");
    } catch (Exception e) {
        System.out.print(e);
    }

    conn.setRequestProperty("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
    conn.setRequestProperty("User-Agent",
            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36");
    conn.setRequestProperty("Accept-Encoding", "gzip, deflate, br");
    conn.setRequestProperty("Accept-Language", "en-US,en;q=0.8");
    // conn.setRequestProperty("Cookie",cookie);
    for (String cookie : cookies) {
        conn.addRequestProperty("Cookie", cookie.split(";", 2)[0]);
    }
    conn.setDoInput(true);
    conn.setDoOutput(true);

    try {

        conn.connect();
        inputStream = conn.getInputStream();

        document = IOUtils.toString(inputStream, "UTF-8");

    } catch (Exception e) {
        System.out.print(e);
    }

    for (String cookie : cookies) {
        System.out.println("cookie -- " + cookie);
    }

    System.out.println("cookie2 -- " + cookie2);

    return document;
}
}