Below is the source of two Java classes I wrote a while ago. The purpose of these classes is to make HTTP web requests in Java by means of a HttpWrapper object. The Cookie class is used by the HttpWrapper class to store cookies per HttpWrapper object (rather than per domain, which would be advisory). A simple example of use:

HttpWrapper wrapper = new HttpWrapper();
wrapper.get("http://www.google.com/");
assert(wrapper.getHtml().contains("Google"));

I’m aware that it is flawed, both in logic as well as in security (cookies are sent cross-domain, see above), but the goal of all this is to allow Java programs to mimic some basic browser behavior where wanted. That is also the reason that this class spoofs the Mozilla user-agent headers.

Feel free to use and modify.

Download the jar file here.
Find the documentation here.

HttpWrapper.java

package honoki.web;

import java.io.*;
import java.net.*;

/**
 * This class provides an environment to easily make web requests (both GET and POST) to a server.
 * It supports the use of cookies and proxies.
 *
 * @author Pieter Hiele
 * @version 2.0
 */
public class HttpWrapper {

  /**
   * Default constructor of a HttpWrapper.
   */
  public HttpWrapper() {
    cookie = new Cookie();
    html = "";
    this.noProxy();
  }

  /**
   * A method to get the content of a given page, with a given referer.
   *
   * @param url
   * 			The given URL.
   * @param referer
   * 			The given referer.
   * @throws IllegalStateException
   * 			Whenever an IO-related problem occurs.
   * @post
   * 			new.getHtml() will return the headers and content of the given URL.
   */
  public void get(String url, String referer) {
    try {
      URL url_ = new URL(url);
      HttpURLConnection conn;
      conn = (HttpURLConnection) url_.openConnection(getProxy());
      conn.setRequestMethod("GET");
      conn.setAllowUserInteraction(false);
      conn.setDoOutput(false);
      conn.setInstanceFollowRedirects(false);

      conn.setRequestProperty("User-Agent", "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9) Gecko/2008061015 Firefox/3.0");
      conn.setRequestProperty("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
      conn.setRequestProperty("Accept-Language", "en-us,en;q=0.5");
      //conn.setRequestProperty("Accept-Encoding", "gzip,deflate");
      conn.setRequestProperty("Accept-Charset", "ISO-8859-1,utf-8;q=0.7,*;q=0.7");
      conn.setRequestProperty("Keep-Alive","300");
      conn.setRequestProperty("Connection", "keep-alive");
      conn.setRequestProperty("Referer", referer);
      if(!cookie.toString().isEmpty()) conn.setRequestProperty("Cookie", cookie.toString());

          // Get response-headers
          String headers = "";
          for(String key: conn.getHeaderFields().keySet())
          	headers += ((key != null)?key + ": ":"") + conn.getHeaderField(key) + "\n";

      // Get content
      BufferedReader d = new BufferedReader(new InputStreamReader(new DataInputStream(conn.getInputStream())));
      String result = "";
      String line = null;
      while ((line = d.readLine()) != null) result += line +"\n";
      d.close();

      cookie.setCookies(conn);
      setLastPage(url);

      setHtml(headers + "\n" + result);
    } catch (IOException e) {
      throw new IllegalStateException("An IOException occurred:\n"+e.getMessage());
    }
  }

  /**
   * A method to get headers and content of a given URL without specifying the referer.
   *
   * @param url
   * 			The given URL.
   * @throws IllegalStateException
   * 			Whenever an IO-related problem occurs.
   * @effect
   * 			| get(url, getLastPage())
   * @see
   * 			{@link #get(String, String)}
   */
  public void get(String url) throws IllegalStateException {
    get(url, getLastPage());
  }

  /**
   * Post the given postdata to a given url with a given referer, and return the response of the server.
   *
   * @param url
   * 			The given URL.
   * @param postdata
   * 			The given postdata.
   * @param referer
   * 			The given referer.
   * @throws IllegalStateException
   * 			Whenever a IO-related problem occurs.
   * @post
   * 			new.getHtml() will return the response of the web server on this request.
   */
  public void post(String url, String postdata, String referer) throws IllegalStateException {
    try {
      URL url_ = new URL(url);
      HttpURLConnection conn = (HttpURLConnection) url_.openConnection(getProxy());

      conn.setRequestMethod("POST");
      conn.setAllowUserInteraction(false);
      conn.setDoOutput(true);
      conn.setInstanceFollowRedirects(false);

      conn.setRequestProperty("User-Agent", "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9) Gecko/2008061015 Firefox/3.0");
      conn.setRequestProperty("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
      conn.setRequestProperty("Accept-Language", "en-us,en;q=0.5");
      //conn.setRequestProperty("Accept-Encoding", "gzip,deflate");
      conn.setRequestProperty("Accept-Charset", "ISO-8859-1,utf-8;q=0.7,*;q=0.7");
      conn.setRequestProperty("Keep-Alive","300");
      conn.setRequestProperty("Connection", "keep-alive");
      conn.setRequestProperty("Referer", referer);
      if(!cookie.toString().isEmpty()) conn.setRequestProperty("Cookie", cookie.toString());
      conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
      conn.setRequestProperty("Content-Length", Integer.toString(postdata.length()));

      // Write postdata
      OutputStreamWriter wr = new OutputStreamWriter(conn.getOutputStream());
          wr.write(postdata);
          wr.flush();
          wr.close();

          // Get response-headers
          String headers = "";
          for(String key: conn.getHeaderFields().keySet())
          	headers += ((key != null)?key + ": ":"") + conn.getHeaderField(key) + "\n";

          // Get content
      String result = "";
      String line = null;
      BufferedReader d = new BufferedReader(new InputStreamReader(new DataInputStream(conn.getInputStream())));
      while ((line = d.readLine()) != null) result += line +"\n";
      d.close();

      cookie.setCookies(conn);
      setLastPage(url);

      setHtml(headers + "\n" + result);
    } catch (IOException e) {
      throw new IllegalStateException("An IOException occurred:\n"+e.getMessage());
    }
  }

  /**
   * A method to get the response of the web server after requesting the given URL without specifying the referer.
   *
   * @param url
   * 			The given URL.
   * @param postdata
   * 			The given postdata.
   * @throws IllegalStateException
   * 			Whenever a IO-related problem occurs.
   * @effect
   * 			| post(url, postdata, getLastPage())
   * @see
   * 			{@link #post(String, String, String)}
   */
  public void post(String url, String postdata) throws IllegalStateException {
    post(url, postdata, getLastPage());
  }

  /**
   * Set the value of the last page that was visited.
   *
   * @param lastpage
   * 			The new value of the last page.
   * @throws IllegalArgumentException
   * 			If the given page is null.
   * @post
   * 			| new.getLastPage().equals(lastpage);
   */
  public void setLastPage(String lastpage) {
    if(lastpage == null) throw new IllegalArgumentException("The given page must not be null.");
    this.lastpage = lastpage;
  }

  /**
   * Return the last page that was visited.
   */
  public String getLastPage() {
    return lastpage;
  }

  /**
   * Clear the last page that was visited.
   *
   * @effect	| setLastPage("");
   */
  public void clearLastPage() {
    setLastPage("");
  }

  /**
   * The page that was last visited.
   */
  private String lastpage = "";

  /**
   * Set a custom cookie for this HttpWrapper.
   *
   * @param cookie	The given cookie.
   * @post			| new.getCookie().equals(cookie);
   */
  public void setCookie(Cookie cookie) {
    if(cookie == null) throw new IllegalArgumentException("The given cookie must not be null.");
    this.cookie = cookie;
  }

  /**
   * Return a clone of the Cookie of this HttpWrapper.
   */
  public Cookie getCookie() {
    return cookie.clone();
  }

  /**
   * The cookies of this wrapper.
   * Be aware that the cookies are not stored per domain name, but per instance of a HttpWrapper.
   */
  private Cookie cookie;

  /**
   * Return the html content that this Wrapper has last retrieved from a request.
   */
  public String getHtml() {
    return this.html;
  }

  /**
   * Set the html content of this HttpWrapper.
   *
   * @param html
   * 			The new html content.
   */
  private void setHtml(String html) {
    this.html = html;
  }

  /**
   * The html content of the last page this HttpWrapper requested.
   */
  private String html;

  /**
   * Set a proxy for this HttpWrapper.
   *
   * @param host
   * 			The host of the proxy.
   * @param port
   * 			The port of the proxy.
   * @see
   * 			{@link #noProxy()} to disable the use of a proxy.
   */
  public void setProxy(String host, int port) {
    this.proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(host, port));
  }

  /**
   * Disable the use of a proxy for this HttpWrapper.
   */
  public void noProxy() {
    this.proxy = Proxy.NO_PROXY;
  }

  /**
   * Return the proxy of this HttpWrapper.
   */
  private Proxy getProxy() {
    return this.proxy;
  }

  /**
   * The proxy to be used with this wrapper.
   */
  private Proxy proxy;

}

Cookie.java

package honoki.web;

import java.io.Serializable;
import java.net.URLConnection;
import java.util.Enumeration;
import java.util.Hashtable;
import java.util.StringTokenizer;

/**
 * This class describes a cookie. It is designed to work with {@link honoki.web.HttpWrapper}.
 * A Cookie instance of this class does not differentiate between different domains,
 * so the user is expected to handle the cookies appropriately.
 *
 * @author Pieter Hiele
 * @version 1.1
 */
public class Cookie implements Serializable {

  private static final long serialVersionUID = 1L;

  /**
   * The default constructor.
   */
  public Cookie() { }

  /**
   * This constructor allows you to provide a textual cookie that will be stored.
   *
   * @param cookie
   * 			The cookie to be stored.
   */
  public Cookie(String cookie) {
    setCookies(cookie);
  }

  /**
   * Read the cookies from the given connection and save them in this Cookie.
   *
   * @param conn
   * 			The given connection to extract the cookies from.
   */
  public void setCookies(URLConnection conn) {
    int i=1;
    String hdrKey, hdrString, aCookie;
    while ((hdrKey = conn.getHeaderFieldKey(i)) != null) {
      if (hdrKey.equals("Set-Cookie")) {
        hdrString = conn.getHeaderField(i);
        StringTokenizer st = new StringTokenizer(hdrString, ",");
        while (st.hasMoreTokens()) {
          String s = st.nextToken();
          aCookie = (s.indexOf(";") > -1) ? s.substring(0, s.indexOf(";")) : s;
          int j = aCookie.indexOf("=");
          if (j > -1)	cookies.put(aCookie.substring(0, j), aCookie.substring(j + 1));
        }
      }
      i++;
    }
  }

  /**
   * Read the cookies from a string and save them in this Cookie.
   *
   * @param cookie
   * 		A cookie formatted as a string.
   */
  public void setCookies(String cookie) {
    String cookies[] = cookie.split("; ");
    String parts[];
    for(String c: cookies) {
      if(c.contains("=")) {
        parts = c.split("=",2);
        this.cookies.put(parts[0], parts[1]);
      }
    }
  }

  /**
   * Return a textual representation of this Cookie.
   */
  public String toString() {
    String result = "";
    Enumeration keys = cookies.keys();
    while (keys.hasMoreElements()) {
      String key = (String)keys.nextElement();
      result += key + "=" + cookies.get(key);
      if (keys.hasMoreElements()) result += "; ";
    }
    return result;
  }

  /**
   * Clear the current cookie of this wrapper.
   *
   * @post
   * 			| new.getCookies().equals("")
   */
  public void clearCookies() {
    cookies.clear();
  }

  /**
   * Remove the cookie with the given name.
   *
   * @param name
   * 		The name of the cookie to remove.
   */
  public void removeCookie(String name) {
    cookies.remove(name);
  }

  /**
   * The actual data structure to store the cookie parts in.
   */
  private Hashtable cookies = new Hashtable();

  /**
   * Returns a clone of this Cookie.
   */
  @Override
  public Cookie clone() {
    Cookie clone = new Cookie();
    clone.setCookies(this.toString());
    return clone;
  }

}