archive/releng.builder/tools/wtp-bug2html/src/bug2html/BugFetcher.java - gerrit/webtools/webtools.releng - Git at Google

 /*
  * Copyright (c) 2005, Innoopract Informationssysteme GmbH.
  * All rights reserved. This program and the accompanying materials
  * are made available under the terms of the Eclipse Public License v1.0
  * which accompanies this distribution, and is available at
  * http://www.eclipse.org/legal/epl-v10.html
  *
  * Contributors:
  *  Innoopract - initial API and implementation
  */
 package bug2html;

 import java.io.BufferedInputStream;
 import java.io.BufferedReader;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.OutputStreamWriter;
 import java.io.UnsupportedEncodingException;
 import java.net.URL;
 import java.net.URLConnection;
 import java.security.KeyManagementException;
 import java.security.NoSuchAlgorithmException;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.Set;
 import java.util.TreeSet;
 import javax.net.ssl.HttpsURLConnection;
 import javax.net.ssl.SSLContext;
 import javax.net.ssl.TrustManager;
 import javax.xml.parsers.DocumentBuilder;
 import javax.xml.parsers.DocumentBuilderFactory;
 import javax.xml.parsers.ParserConfigurationException;
 import org.w3c.dom.Document;
 import org.w3c.dom.NodeList;
 import org.xml.sax.SAXException;

 /**
  * Queries Bugzilla for enhancements and writes result as an XML milestone plan.
  *
  * @author Elias Volanakis <evolanakis@innoopract.de>
  *
  */
 public class BugFetcher {

   /** Bugzilla query, which shows a list of Bugs in RDF format. */
   // This query fetches NEW enchancements of the Webtools Project
   private static final String BUGZILLA_RDF_QUERY = "https://bugs.eclipse.org/bugs/buglist.cgi?query_format=advanced&short_desc_type=allwordssubstr&short_desc=&product=Web%20Tools&long_desc_type=allwordssubstr&long_desc=&bug_file_loc_type=allwordssubstr&bug_file_loc=&keywords_type=allwords&keywords=&bug_status=NEW&bug_severity=enhancement&emailtype1=substring&email1=&emailtype2=substring&email2=&bugidtype=include&bug_id=&votes=&chfieldfrom=&chfieldto=Now&chfieldvalue=&field0-0-0=noop&type0-0-0=noop&value0-0-0=&format=rdf";
   // This query fetches all enchancements of the Webtools Project
   // private static final String BUGZILLA_RDF_QUERY = "https://bugs.eclipse.org/bugs/buglist.cgi?query_format=advanced&short_desc_type=allwordssubstr&short_desc=&product=Web%20Tools&long_desc_type=allwordssubstr&long_desc=&bug_file_loc_type=allwordssubstr&bug_file_loc=&keywords_type=allwords&keywords=&bug_status=NEW&bug_status=ASSIGNED&bug_status=REOPENED&bug_severity=enhancement&emailtype1=substring&email1=&emailtype2=substring&email2=&bugidtype=include&bug_id=&votes=&chfieldfrom=&chfieldto=Now&chfieldvalue=&field0-0-0=noop&type0-0-0=noop&value0-0-0=&format=rdf";

   /** URL prefix for showing a single Bug */
   public static final String BUGZILLA_BUGLIST_QUERY = "https://bugs.eclipse.org/bugs/show_bug.cgi?";

   private static DocumentBuilder builder;

   private static TrustManager[] trustManagers;

   private BugFetcherOptions options;

   BugFetcher(final String[] args) {
     options = new BugFetcherOptions(args);
   }

   BugFetcherOptions getOptions() {
     return options;
   }

   private void execute() throws IOException {
     if (getOptions().isHelpWanted()) {
       getOptions().printUsage();
     } else {
       Set bugs = fetchBugList();

       if (bugs.size() == 0) {
         System.err.println("Your bugzilla query didn't return any bugs.");
         System.err.println("Aborting");
         System.err.println("Query:" + BUGZILLA_RDF_QUERY);
       } else {
         String query = constructBugQuery(bugs);
         System.out.println(query);
         bugs = fetchBugs(query);

         XMLGenerator generator = new XMLGenerator();
         generator.setEmptyMilestones(getOptions().isEmptyMilestones());
         StringBuffer xml = generator.generateXML(bugs);

         String filename = "enhancement_overview.xml";
         System.out.println("Saving to: " + filename);
         OutputStreamWriter out = new OutputStreamWriter(new FileOutputStream(
             new File(filename)), "ISO-8859-1");
         out.write(xml.toString());
         out.flush();
         out.close();
       }
     }
   }

   public static void main(final String[] args) throws IOException {
     BugFetcher fetcher = new BugFetcher(args);
     fetcher.execute();
   }

   private Document connectAndParse(final String uri) {
     BufferedInputStream is = null;
     Document result = null;
     try {
       disableCertificates();
       is = connectTo(uri);
       // isoToUTF is a workaround to a Bugzilla-Bug,
       // where the XML returned contains non UTF-8 characters
       result = parseXML(isoToUTF(is));
     } catch (IOException ioe) {
       ioe.printStackTrace();
     } catch (SAXException se) {
       se.printStackTrace();
     } finally {
       tryClose(is);
     }
     return result;
   }

   private BufferedInputStream connectTo(final String uri) throws IOException {
     URL url = new URL(uri);
     URLConnection connection = url.openConnection();
     BufferedInputStream result = new BufferedInputStream(connection
         .getInputStream());
     return result;
   }

   private String constructBugQuery(final Set bugs) {
     if (bugs == null || bugs.size() == 0) {
       throw new IllegalArgumentException();
     }
     Iterator iter = bugs.iterator();
     StringBuffer queryString = new StringBuffer();
     queryString.append(BUGZILLA_BUGLIST_QUERY);
     queryString.append("ctype=xml");
     while (iter.hasNext()) {
       String bugId = (String) iter.next();
       queryString.append("&id=" + bugId);
     }
     return queryString.toString();
   }

   private void disableCertificates() {
     try {
       if (trustManagers == null) {
         trustManagers = new TrustManager[1];
         trustManagers[0] = new DummyX509TrustManager();
       }
       SSLContext context = SSLContext.getInstance("SSLv3");
       context.init(null, trustManagers, null);
       HttpsURLConnection.setDefaultSSLSocketFactory(context.getSocketFactory());
     } catch (KeyManagementException e) {
       e.printStackTrace();
     } catch (NoSuchAlgorithmException e) {
       e.printStackTrace();
     }
   }

   private Set fetchBugList() {
     StringBuffer query = new StringBuffer(BUGZILLA_RDF_QUERY);
     String[] versions = getOptions().getVersions();
     for (int i = 0; i < versions.length; i++) {
       query.append("&version=" + versions[i]);
     }

     Document bugList = connectAndParse(query.toString());
     NodeList idNodes = bugList.getElementsByTagName("bz:id");
     int nodeCount = idNodes.getLength();
     System.out.println(nodeCount + " bz:id nodes");

     Set result = new HashSet(nodeCount);
     for (int i = 0; i < nodeCount; i++) {
       String bugId = idNodes.item(i).getChildNodes().item(0).getNodeValue();
       result.add(bugId);
     }

     return result;
   }

   private Set fetchBugs(final String bugzillaQuery) throws IOException {
     Document bugList = connectAndParse(bugzillaQuery);
     NodeList bugNodes = bugList.getElementsByTagName("bug");
     int nodeCount = bugNodes.getLength();
     System.out.println(nodeCount + " bug nodes");

     Set result = new TreeSet(new BugComparator());
     for (int i = 0; i < nodeCount; i++) {
       result.add(BugFactory.newBug(bugNodes.item(i)));
     }
     return result;
   }

   private DocumentBuilder getBuilder() {
     if (builder == null) {
       try {
         builder = DocumentBuilderFactory.newInstance().newDocumentBuilder();
       } catch (ParserConfigurationException e) {
         e.printStackTrace();
       }
     }
     return builder;
   }

   private Document parseXML(final InputStream is) throws IOException,
       SAXException {
     if (is == null) {
       throw new IllegalArgumentException();
     }
     Document result = null;
     result = getBuilder().parse(is);
     return result;
   }

   /**
    * Converts ISO-8859-1 to UTF-8.
    * Note: Straightforward but inefficient implementation.
    */
   private InputStream isoToUTF(final BufferedInputStream bis) {
     BufferedReader isoReader = null;
     OutputStreamWriter utfWriter = null;
     ByteArrayOutputStream outBuffer = new ByteArrayOutputStream();
     ByteArrayInputStream inBuffer = null;

     try {
       isoReader = new BufferedReader(new InputStreamReader(bis, "ISO-8859-1"));
       utfWriter = new OutputStreamWriter(outBuffer, "UTF-8");
     } catch (UnsupportedEncodingException e) {
       e.printStackTrace(); // should never happen
     }

     try {
       String line;
       while ((line = isoReader.readLine()) != null) {
         utfWriter.write(line);
         utfWriter.write("\n");
       }
       utfWriter.flush();
       inBuffer = new ByteArrayInputStream(outBuffer.toByteArray());
     } catch (IOException e) {
       e.printStackTrace();
     } finally {
       tryClose(utfWriter);
     }

     return inBuffer;
   }

   private void tryClose(final BufferedInputStream is) {
     if (is != null) {
       try {
         is.close();
       } catch (IOException e) {
         e.printStackTrace();
       }
     }
   }

   private void tryClose(final OutputStreamWriter os) {
     if (os != null) {
       try {
         os.close();
       } catch (IOException e) {
         e.printStackTrace();
       }
     }
   }
 }
	/*
	* Copyright (c) 2005, Innoopract Informationssysteme GmbH.
	* All rights reserved. This program and the accompanying materials
	* are made available under the terms of the Eclipse Public License v1.0
	* which accompanies this distribution, and is available at
	* http://www.eclipse.org/legal/epl-v10.html
	*
	* Contributors:
	* Innoopract - initial API and implementation
	*/
	package bug2html;

	import java.io.BufferedInputStream;
	import java.io.BufferedReader;
	import java.io.ByteArrayInputStream;
	import java.io.ByteArrayOutputStream;
	import java.io.File;
	import java.io.FileOutputStream;
	import java.io.IOException;
	import java.io.InputStream;
	import java.io.InputStreamReader;
	import java.io.OutputStreamWriter;
	import java.io.UnsupportedEncodingException;
	import java.net.URL;
	import java.net.URLConnection;
	import java.security.KeyManagementException;
	import java.security.NoSuchAlgorithmException;
	import java.util.HashSet;
	import java.util.Iterator;
	import java.util.Set;
	import java.util.TreeSet;
	import javax.net.ssl.HttpsURLConnection;
	import javax.net.ssl.SSLContext;
	import javax.net.ssl.TrustManager;
	import javax.xml.parsers.DocumentBuilder;
	import javax.xml.parsers.DocumentBuilderFactory;
	import javax.xml.parsers.ParserConfigurationException;
	import org.w3c.dom.Document;
	import org.w3c.dom.NodeList;
	import org.xml.sax.SAXException;

	/**
	* Queries Bugzilla for enhancements and writes result as an XML milestone plan.
	*
	* @author Elias Volanakis <evolanakis@innoopract.de>
	*
	*/
	public class BugFetcher {

	/** Bugzilla query, which shows a list of Bugs in RDF format. */
	// This query fetches NEW enchancements of the Webtools Project
	private static final String BUGZILLA_RDF_QUERY = "https://bugs.eclipse.org/bugs/buglist.cgi?query_format=advanced&short_desc_type=allwordssubstr&short_desc=&product=Web%20Tools&long_desc_type=allwordssubstr&long_desc=&bug_file_loc_type=allwordssubstr&bug_file_loc=&keywords_type=allwords&keywords=&bug_status=NEW&bug_severity=enhancement&emailtype1=substring&email1=&emailtype2=substring&email2=&bugidtype=include&bug_id=&votes=&chfieldfrom=&chfieldto=Now&chfieldvalue=&field0-0-0=noop&type0-0-0=noop&value0-0-0=&format=rdf";
	// This query fetches all enchancements of the Webtools Project
	// private static final String BUGZILLA_RDF_QUERY = "https://bugs.eclipse.org/bugs/buglist.cgi?query_format=advanced&short_desc_type=allwordssubstr&short_desc=&product=Web%20Tools&long_desc_type=allwordssubstr&long_desc=&bug_file_loc_type=allwordssubstr&bug_file_loc=&keywords_type=allwords&keywords=&bug_status=NEW&bug_status=ASSIGNED&bug_status=REOPENED&bug_severity=enhancement&emailtype1=substring&email1=&emailtype2=substring&email2=&bugidtype=include&bug_id=&votes=&chfieldfrom=&chfieldto=Now&chfieldvalue=&field0-0-0=noop&type0-0-0=noop&value0-0-0=&format=rdf";

	/** URL prefix for showing a single Bug */
	public static final String BUGZILLA_BUGLIST_QUERY = "https://bugs.eclipse.org/bugs/show_bug.cgi?";

	private static DocumentBuilder builder;

	private static TrustManager[] trustManagers;

	private BugFetcherOptions options;

	BugFetcher(final String[] args) {
	options = new BugFetcherOptions(args);
	}

	BugFetcherOptions getOptions() {
	return options;
	}

	private void execute() throws IOException {
	if (getOptions().isHelpWanted()) {
	getOptions().printUsage();
	} else {
	Set bugs = fetchBugList();

	if (bugs.size() == 0) {
	System.err.println("Your bugzilla query didn't return any bugs.");
	System.err.println("Aborting");
	System.err.println("Query:" + BUGZILLA_RDF_QUERY);
	} else {
	String query = constructBugQuery(bugs);
	System.out.println(query);
	bugs = fetchBugs(query);

	XMLGenerator generator = new XMLGenerator();
	generator.setEmptyMilestones(getOptions().isEmptyMilestones());
	StringBuffer xml = generator.generateXML(bugs);

	String filename = "enhancement_overview.xml";
	System.out.println("Saving to: " + filename);
	OutputStreamWriter out = new OutputStreamWriter(new FileOutputStream(
	new File(filename)), "ISO-8859-1");
	out.write(xml.toString());
	out.flush();
	out.close();
	}
	}
	}

	public static void main(final String[] args) throws IOException {
	BugFetcher fetcher = new BugFetcher(args);
	fetcher.execute();
	}

	private Document connectAndParse(final String uri) {
	BufferedInputStream is = null;
	Document result = null;
	try {
	disableCertificates();
	is = connectTo(uri);
	// isoToUTF is a workaround to a Bugzilla-Bug,
	// where the XML returned contains non UTF-8 characters
	result = parseXML(isoToUTF(is));
	} catch (IOException ioe) {
	ioe.printStackTrace();
	} catch (SAXException se) {
	se.printStackTrace();
	} finally {
	tryClose(is);
	}
	return result;
	}

	private BufferedInputStream connectTo(final String uri) throws IOException {
	URL url = new URL(uri);
	URLConnection connection = url.openConnection();
	BufferedInputStream result = new BufferedInputStream(connection
	.getInputStream());
	return result;
	}

	private String constructBugQuery(final Set bugs) {
	if (bugs == null \|\| bugs.size() == 0) {
	throw new IllegalArgumentException();
	}
	Iterator iter = bugs.iterator();
	StringBuffer queryString = new StringBuffer();
	queryString.append(BUGZILLA_BUGLIST_QUERY);
	queryString.append("ctype=xml");
	while (iter.hasNext()) {
	String bugId = (String) iter.next();
	queryString.append("&id=" + bugId);
	}
	return queryString.toString();
	}

	private void disableCertificates() {
	try {
	if (trustManagers == null) {
	trustManagers = new TrustManager[1];
	trustManagers[0] = new DummyX509TrustManager();
	}
	SSLContext context = SSLContext.getInstance("SSLv3");
	context.init(null, trustManagers, null);
	HttpsURLConnection.setDefaultSSLSocketFactory(context.getSocketFactory());
	} catch (KeyManagementException e) {
	e.printStackTrace();
	} catch (NoSuchAlgorithmException e) {
	e.printStackTrace();
	}
	}

	private Set fetchBugList() {
	StringBuffer query = new StringBuffer(BUGZILLA_RDF_QUERY);
	String[] versions = getOptions().getVersions();
	for (int i = 0; i < versions.length; i++) {
	query.append("&version=" + versions[i]);
	}

	Document bugList = connectAndParse(query.toString());
	NodeList idNodes = bugList.getElementsByTagName("bz:id");
	int nodeCount = idNodes.getLength();
	System.out.println(nodeCount + " bz:id nodes");

	Set result = new HashSet(nodeCount);
	for (int i = 0; i < nodeCount; i++) {
	String bugId = idNodes.item(i).getChildNodes().item(0).getNodeValue();
	result.add(bugId);
	}

	return result;
	}

	private Set fetchBugs(final String bugzillaQuery) throws IOException {
	Document bugList = connectAndParse(bugzillaQuery);
	NodeList bugNodes = bugList.getElementsByTagName("bug");
	int nodeCount = bugNodes.getLength();
	System.out.println(nodeCount + " bug nodes");

	Set result = new TreeSet(new BugComparator());
	for (int i = 0; i < nodeCount; i++) {
	result.add(BugFactory.newBug(bugNodes.item(i)));
	}
	return result;
	}

	private DocumentBuilder getBuilder() {
	if (builder == null) {
	try {
	builder = DocumentBuilderFactory.newInstance().newDocumentBuilder();
	} catch (ParserConfigurationException e) {
	e.printStackTrace();
	}
	}
	return builder;
	}

	private Document parseXML(final InputStream is) throws IOException,
	SAXException {
	if (is == null) {
	throw new IllegalArgumentException();
	}
	Document result = null;
	result = getBuilder().parse(is);
	return result;
	}

	/**
	* Converts ISO-8859-1 to UTF-8.
	* Note: Straightforward but inefficient implementation.
	*/
	private InputStream isoToUTF(final BufferedInputStream bis) {
	BufferedReader isoReader = null;
	OutputStreamWriter utfWriter = null;
	ByteArrayOutputStream outBuffer = new ByteArrayOutputStream();
	ByteArrayInputStream inBuffer = null;

	try {
	isoReader = new BufferedReader(new InputStreamReader(bis, "ISO-8859-1"));
	utfWriter = new OutputStreamWriter(outBuffer, "UTF-8");
	} catch (UnsupportedEncodingException e) {
	e.printStackTrace(); // should never happen
	}

	try {
	String line;
	while ((line = isoReader.readLine()) != null) {
	utfWriter.write(line);
	utfWriter.write("\n");
	}
	utfWriter.flush();
	inBuffer = new ByteArrayInputStream(outBuffer.toByteArray());
	} catch (IOException e) {
	e.printStackTrace();
	} finally {
	tryClose(utfWriter);
	}

	return inBuffer;
	}

	private void tryClose(final BufferedInputStream is) {
	if (is != null) {
	try {
	is.close();
	} catch (IOException e) {
	e.printStackTrace();
	}
	}
	}

	private void tryClose(final OutputStreamWriter os) {
	if (os != null) {
	try {
	os.close();
	} catch (IOException e) {
	e.printStackTrace();
	}
	}
	}
	}