Adam Pritchard avatar Adam Pritchard committed 872edfa Merge

Merged diagnostic-feeback branch

Comments (0)

Files changed (8)

Android/PsiphonAndroid/.classpath

 	<classpathentry kind="con" path="com.android.ide.eclipse.adt.LIBRARIES"/>
 	<classpathentry kind="src" path="src"/>
 	<classpathentry kind="src" path="gen"/>
+	<classpathentry kind="lib" path="libs/snakeyaml-1.10-android.jar"/>
 	<classpathentry kind="output" path="bin/classes"/>
 </classpath>
Add a comment to this file

Android/PsiphonAndroid/libs/snakeyaml-1.10-android.jar

Binary file added.

Android/PsiphonAndroid/src/com/psiphon3/FeedbackActivity.java

 import java.security.PublicKey;
 import java.security.SecureRandom;
 import java.security.spec.X509EncodedKeySpec;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
 import java.util.Locale;
+import java.util.Map;
 
 import javax.crypto.Cipher;
 import javax.crypto.KeyGenerator;
 import javax.crypto.SecretKey;
 import javax.crypto.spec.IvParameterSpec;
 
+import org.yaml.snakeyaml.Yaml;
+
 import com.psiphon3.PsiphonData.StatusEntry;
 import com.psiphon3.ServerInterface.PsiphonServerInterfaceException;
 import com.psiphon3.Utils.Base64;
             private File createEmailAttachment()
             {
                 // Our attachment is YAML, which is then encrypted, and the 
-                // encryption elements stored in JSON.                
+                // encryption elements stored in JSON.
                 
-                StringBuilder content = new StringBuilder();
-
-                content.append("--- # System Info\n\n");
-                content.append("Build:\n");
-                content.append("  BRAND: ").append(Build.BRAND).append("\n");
-                content.append("  CPU_ABI: ").append(Build.CPU_ABI).append("\n");
-                content.append("  MANUFACTURER: ").append(Build.MANUFACTURER).append("\n");
-                content.append("  MODEL: ").append(Build.MODEL).append("\n");
-                content.append("  TAGS: ").append(Build.TAGS).append("\n");
-                content.append("  VERSION.CODENAME: ").append(Build.VERSION.CODENAME).append("\n");
-                content.append("  VERSION.RELEASE: ").append(Build.VERSION.RELEASE).append("\n");
-                content.append("  VERSION.SDK_INT: ").append(Build.VERSION.SDK_INT).append("\n");
-                content.append("isRooted: ").append(Utils.isRooted()).append("\n");
-                content.append("psiphonEmbeddedValues:\n");
-                content.append("  PROPAGATION_CHANNEL_ID: ").append(EmbeddedValues.PROPAGATION_CHANNEL_ID).append("\n");
-                content.append("  SPONSOR_ID: ").append(EmbeddedValues.SPONSOR_ID).append("\n");
-                content.append("  CLIENT_VERSION: ").append(EmbeddedValues.CLIENT_VERSION).append("\n");
-                content.append("\n");
+                SimpleDateFormat dateParser = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
                 
-                content.append("--- # Server Response Check\n\n");
-                ArrayList<PsiphonData.ServerResponseCheck> serverResponseChecks = PsiphonData.cloneServerResponseChecks();
-                for (PsiphonData.ServerResponseCheck entry : serverResponseChecks)
+                String diagnosticYaml;
+                try
                 {
-                    content.append("- ipAddress: \"").append(entry.ipAddress).append("\"\n");
-                    content.append("  responded: ").append(entry.responded).append("\n");
-                    content.append("  responseTime: ").append(entry.responseTime).append("\n");
-                }
-                content.append("\n");
-
-                ArrayList<String> diagnosticHistory = PsiphonData.cloneDiagnosticHistory();
-                if (!diagnosticHistory.isEmpty())
-                {
-                    content.append("--- # Diagnostic History\n\n");
-                    for (String entry : diagnosticHistory)
+                    /*
+                     * System Information
+                     */
+                    
+                    Map<String, Object> sysInfo = new HashMap<String, Object>();
+                    Map<String, Object> sysInfo_Build = new HashMap<String, Object>();
+                    sysInfo.put("Build", sysInfo_Build);
+                    sysInfo_Build.put("BRAND", Build.BRAND);
+                    sysInfo_Build.put("CPU_ABI", Build.CPU_ABI);
+                    sysInfo_Build.put("MANUFACTURER", Build.MANUFACTURER);
+                    sysInfo_Build.put("MODEL", Build.MODEL);
+                    sysInfo_Build.put("TAGS", Build.TAGS);
+                    sysInfo_Build.put("VERSION.CODENAME", Build.VERSION.CODENAME);
+                    sysInfo_Build.put("VERSION.RELEASE", Build.VERSION.RELEASE);
+                    sysInfo_Build.put("VERSION.SDK_INT", Build.VERSION.SDK_INT);
+                    sysInfo.put("isRooted", Utils.isRooted());
+                    Map<String, Object> sysInfo_psiphonEmbeddedValues = new HashMap<String, Object>();
+                    sysInfo.put("psiphonEmbeddedValues", sysInfo_psiphonEmbeddedValues);
+                    sysInfo_psiphonEmbeddedValues.put("PROPAGATION_CHANNEL_ID", EmbeddedValues.PROPAGATION_CHANNEL_ID);
+                    sysInfo_psiphonEmbeddedValues.put("SPONSOR_ID", EmbeddedValues.SPONSOR_ID);
+                    sysInfo_psiphonEmbeddedValues.put("CLIENT_VERSION", EmbeddedValues.CLIENT_VERSION);
+                    
+                    /*
+                     * Server Response Check
+                     */
+                    
+                    List<Object> serverResponseChecks = new ArrayList<Object>();
+                    for (PsiphonData.ServerResponseCheck item : PsiphonData.cloneServerResponseChecks())
                     {
-                        content.append("- \"").append(entry).append("\"\n");
+                        Map<String, Object> entry = new HashMap<String, Object>();
+                        entry.put("ipAddress", item.ipAddress);
+                        entry.put("responded", item.responded);
+                        entry.put("responseTime", item.responseTime);
+                        entry.put("timestamp", dateParser.parse(item.timestamp));
+                        
+                        serverResponseChecks.add(entry);
                     }
-                    content.append("\n");
-                }
-
-                content.append("--- # Status History\n\n");
-                ArrayList<StatusEntry> history = PsiphonData.cloneStatusHistory();
-                for (StatusEntry entry : history)
-                {
-                    // Don't send any sensitive logs
-                    if (entry.sensitivity == MyLog.Sensitivity.SENSITIVE_LOG)
+    
+                    /*
+                     * Diagnostic History
+                     */
+                    
+                    List<Object> diagnosticHistory = new ArrayList<Object>();
+    
+                    for (PsiphonData.DiagnosticEntry item : PsiphonData.cloneDiagnosticHistory())
                     {
-                        continue;
+                        Map<String, Object> entry = new HashMap<String, Object>();
+                        entry.put("timestamp", dateParser.parse(item.timestamp));
+                        entry.put("msg", item.msg);
+                        entry.put("data", item.data);
+                        
+                        diagnosticHistory.add(entry);
                     }
                     
-                    StringBuilder formatArgs = new StringBuilder();
-                    if (entry.formatArgs != null && entry.formatArgs.length > 0
-                        // Don't send any sensitive format args
-                        && entry.sensitivity != MyLog.Sensitivity.SENSITIVE_FORMAT_ARGS)
+                    /* 
+                     * Status History
+                     */
+                    
+                    List<Object> statusHistory = new ArrayList<Object>();
+                    
+                    for (StatusEntry internalEntry : PsiphonData.cloneStatusHistory())
                     {
-                        formatArgs.append("[");
-                        for (int i = 0; i < entry.formatArgs.length; i++)
+                        // Don't send any sensitive logs
+                        if (internalEntry.sensitivity == MyLog.Sensitivity.SENSITIVE_LOG)
                         {
-                            String arg = entry.formatArgs[i].toString();
-                            formatArgs.append("\"").append(arg).append("\"");
-                            if (i < entry.formatArgs.length-1)
+                            continue;
+                        }
+                        
+                        Map<String, Object> statusEntry = new HashMap<String, Object>();
+                        statusHistory.add(statusEntry);
+                        
+                        statusEntry.put("id", internalEntry.idName);
+                        statusEntry.put("timestamp", dateParser.parse(internalEntry.timestamp));
+                        statusEntry.put("priority", internalEntry.priority);
+                        statusEntry.put("formatArgs", null); 
+                        statusEntry.put("throwable", null); 
+                        
+                        if (internalEntry.formatArgs != null && internalEntry.formatArgs.length > 0
+                            // Don't send any sensitive format args
+                            && internalEntry.sensitivity != MyLog.Sensitivity.SENSITIVE_FORMAT_ARGS)
+                        {
+                            statusEntry.put("formatArgs", Arrays.asList(internalEntry.formatArgs));
+                        }
+    
+                        if (internalEntry.throwable != null)
+                        {
+                            Map<String, Object> throwable = new HashMap<String, Object>();
+                            statusEntry.put("throwable", throwable);
+                            
+                            throwable.put("message", internalEntry.throwable.toString());
+                            
+                            List<String> stack = new ArrayList<String>();
+                            throwable.put("stack", stack);
+                            
+                            for (StackTraceElement element : internalEntry.throwable.getStackTrace())
                             {
-                                formatArgs.append(", ");
+                                stack.add(element.toString());
                             }
                         }
-                        formatArgs.append("]");
-                    }
-                    
-                    StringBuilder throwable = new StringBuilder();
-                    if (entry.throwable != null)
-                    {
-                        throwable.append("\n    message: \"").append(entry.throwable.toString()).append("\"");
-                        throwable.append("\n    stack: ");
-                        for (StackTraceElement element : entry.throwable.getStackTrace())
-                        {
-                            throwable.append("\n      - \"").append(element).append("\"");
-                        }
                     }
                     
+                    /*
+                     * YAML-ify the diagnostic info
+                     */
                     
-                    content.append("- id: ").append(entry.idName).append("\n");
-                    content.append("  timestamp: ").append(entry.timestamp).append("\n");
-                    content.append("  formatArgs: ").append(formatArgs).append("\n");
-                    content.append("  throwable: ").append(throwable).append("\n");
+                    List<Object> diagnosticObjects = new ArrayList<Object>();
+                    diagnosticObjects.add(sysInfo);
+                    diagnosticObjects.add(serverResponseChecks);
+                    diagnosticObjects.add(diagnosticHistory);
+                    diagnosticObjects.add(statusHistory);
+                    Yaml yaml = new Yaml();
+                    diagnosticYaml = yaml.dumpAll(diagnosticObjects.iterator());
                 }
-                content.append("\n");
+                catch (ParseException e)
+                {
+                    // Shouldn't happen. Our date formats should be consistent.
+                    assert(false);
+                    return null;
+                }
                 
                 // Encrypt the file contents
                 byte[] contentCiphertext = null, iv = null, 
                     Cipher aesCipher = Cipher.getInstance("AES/CBC/PKCS5Padding");
                     aesCipher.init(Cipher.ENCRYPT_MODE, encryptionKey, ivParamSpec);
                     
-                    contentCiphertext = aesCipher.doFinal(content.toString().getBytes("UTF-8"));
+                    contentCiphertext = aesCipher.doFinal(diagnosticYaml.getBytes("UTF-8"));
                     
                     // Get the IV. (I don't know if it can be different from the
                     // one generated above, but retrieving it here seems safest.)
 
                 ServerInterface serverInterface = new ServerInterface(activity);
                 serverInterface.start();
+                serverInterface.setCurrentServerEntry();
 
                 String formData;
                 try

Android/PsiphonAndroid/src/com/psiphon3/PsiphonData.java

 package com.psiphon3;
 
 import java.util.ArrayList;
-import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
             m_statusHistory.clear();
         }
     }
+    
+    /*
+     * Diagnostic history support
+     */
 
-    static private ArrayList<String> m_diagnosticHistory = new ArrayList<String>();
+    static public class DiagnosticEntry extends Object
+    {
+        String timestamp;
+        String msg;
+        Object data;
+    }
+    
+    static private List<DiagnosticEntry> m_diagnosticHistory = new ArrayList<DiagnosticEntry>();
 
-    static public void addDiagnosticEntry(String entry)
+    static public void addDiagnosticEntry(String msg, Object data)
     {
+        DiagnosticEntry entry = new DiagnosticEntry();
+        entry.timestamp = Utils.getISO8601String();
+        entry.msg = msg;
+        entry.data = data;
         m_diagnosticHistory.add(entry);
     }
     
-    static public ArrayList<String> cloneDiagnosticHistory()
+    static public List<DiagnosticEntry> cloneDiagnosticHistory()
     {
-        ArrayList<String> copy;
+        List<DiagnosticEntry> copy;
         synchronized(m_diagnosticHistory) 
         {
-            copy = new ArrayList<String>(m_diagnosticHistory);
+            copy = new ArrayList<DiagnosticEntry>(m_diagnosticHistory);
         }
         return copy;
     }
         String ipAddress;
         boolean responded;
         long responseTime;
+        String timestamp;
     }
     
     static private ArrayList<ServerResponseCheck> m_serverResponses = new ArrayList<ServerResponseCheck>();
         entry.ipAddress = ipAddress;
         entry.responded = responded;
         entry.responseTime = responseTime;
+        entry.timestamp = Utils.getISO8601String();
         
         synchronized(m_serverResponses) 
         {

Android/PsiphonAndroid/src/com/psiphon3/ServerInterface.java

     
     synchronized ServerEntry getCurrentServerEntry()
     {
+        if (this.currentServerEntry == null)
+        {
+            return null;
+        }
+        
         return this.currentServerEntry.clone();
     }
     

Android/PsiphonAndroid/src/com/psiphon3/TunnelService.java

 package com.psiphon3;
 
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.TimeUnit;
             checkSignals(0);
 
             MyLog.v(R.string.ssh_connecting, MyLog.Sensitivity.NOT_SENSITIVE);
+            
+            Map<String, String> diagnosticData = new HashMap<String, String>();
+            diagnosticData.put("ipAddress", entry.ipAddress);
+            MyLog.g("ConnectingServer", diagnosticData);
+            
             conn = new Connection(entry.ipAddress, entry.sshObfuscatedKey, entry.sshObfuscatedPort);
             Monitor monitor = new Monitor(m_signalQueue);
             conn.connect(

Android/PsiphonAndroid/src/com/psiphon3/Utils.java

          * except it will also be included in the feedback diagnostic attachment.
          * @param msg The message to log.
          */
-        static void g(String msg)
+        static void g(String msg, Object data)
         {
-            PsiphonData.addDiagnosticEntry(msg);
+            PsiphonData.addDiagnosticEntry(msg, data);
+            // We're not logging the `data` at all. In the future we may want to.
             MyLog.d(msg);
         }
 

EmailResponder/FeedbackDecryptor/mailformatter.py

 
 _template = \
 '''
+## SECURITY IMPORTANT: This causes HTML escaping to be applied to all expression
+## tags (${...}) in this template. Because we're output untrusted user-supplied
+## data, this is essential.
+<%page expression_filter="h"/>
+
 <%!
 import yaml
+from operator import itemgetter
+
+
+# To be used to format datetimes
+def timestamp_display(timestamp):
+    return '{:%Y-%m-%dT%H:%M:%S}.{:03}Z'.format(timestamp, timestamp.microsecond/1000)
+
+
+# Returns a tuple of (diff_float, diff_display_string). Arguments must be
+# datetimes. `last_timestamp` may be None.
+def get_timestamp_diff(last_timestamp, timestamp):
+    timestamp_diff_secs = 0.0
+    if last_timestamp:
+        timestamp_diff_secs = (timestamp - last_timestamp).total_seconds()
+    timestamp_diff_str = '{:.3f}'.format(timestamp_diff_secs)
+    return (timestamp_diff_secs, timestamp_diff_str)
 %>
 
 <%
         color: red;
     }
 
-    .status-entry {
+    .status-entry, .diagnostic-entry {
         margin-bottom: 0.3em;
     }
 
         margin-left: 2em;
     }
 
-    .status-entry .timestamp {
+    .timestamp {
         font-size: 0.8em;
         font-family: monospace;
     }
 
-    .status-entry-id {
+    .status-entry-id, .diagnostic-entry-msg {
         font-weight: bold;
     }
 
+    .priority-info {
+        color: green;
+    }
+
+    .priority-error {
+        color: red;
+    }
+
+    .diagnostic-entry-msg {
+        color: purple;
+    }
+
     hr {
         width: 80%;
         border: 0;
-        background-color: lightGrey;
+        background-color: lightGray;
         height: 1px;
     }
 
         text-align: right;
         font-family: monospace;
     }
+
+    .server-response-checks .separated th,
+    .server-response-checks .separated td {
+        border-top: dotted thin gray;
+    }
 </style>
 
+##
+## System Info
+##
+
 <h1>System Info</h1>
 
 ## Display more human-friendly field names
     % endfor
 </table>
 
-<%def name="server_response_row(name, ping)">
+##
+## Server Response Checks
+##
+
+<%def name="server_response_row(entry, last_timestamp)">
     <%
-    ping_class = 'good'
-    ping_str = '%dms' % ping
-    if ping < 0:
-        ping_class = 'bad'
-        ping_str = 'none'
-    elif ping > 2000:
-        ping_class = 'warn'
+        # Put a separator between entries that are separated in time.
+        timestamp_separated_class = ''
+        if last_timestamp and 'timestamp' in entry:
+            if (entry['timestamp'] - last_timestamp).total_seconds() > 20:
+                timestamp_separated_class = 'separated'
+
+        ping_class = 'good'
+        ping_str = '%dms' % entry['responseTime']
+        if entry['responseTime'] < 0:
+            ping_class = 'bad'
+            ping_str = 'none'
+        elif entry['responseTime'] > 2000:
+            ping_class = 'warn'
     %>
-    <tr>
-        <th>${name}</th>
+    <tr class="${timestamp_separated_class}">
+        <th>${entry['ipAddress']}</th>
         <td class="intcompare ${ping_class}">${ping_str}</td>
+        <td class="timestamp">${entry['timestamp'] if 'timestamp' in entry else ''}</td>
     </tr>
 </%def>
 
 <h1>Server Response Checks</h1>
-<table>
-    % for resp in server_responses:
-        ${server_response_row(resp['ipAddress'], resp['responseTime'])}
+<table class="server-response-checks">
+    <% last_timestamp = None %>
+    % for entry in server_responses:
+        ${server_response_row(entry, last_timestamp)}
+        <% last_timestamp = entry['timestamp'] if 'timestamp' in entry else None %>
     % endfor
 </table>
 
-% if diagnostic_history:
-<h1>Diagnostic History</h1>
-<table>
-    % for entry in diagnostic_history:
-        <tr><td>${entry}</td></tr>
-    % endfor
-</table>
-% endif
+##
+## Status History and Diagnostic History
+##
 
 <%def name="status_history_row(entry, last_timestamp)">
     <%
-    timestamp_diff_secs = 0.0
-    if last_timestamp:
-        timestamp_diff_secs = (entry['timestamp'] - last_timestamp).total_seconds()
-    timestamp_diff_str = '{:.3f}'.format(timestamp_diff_secs)
+        timestamp_diff_secs, timestamp_diff_str = get_timestamp_diff(last_timestamp, entry['timestamp'])
 
-    timestamp_str = '{:%Y-%m-%dT%H:%M:%S}.{:03}Z'.format(entry['timestamp'], entry['timestamp'].microsecond/1000)
+        # These values come from the Java definitions for Log.VERBOSE, etc.
+        PRIORITY_CLASSES = {
+            2: 'priority-verbose',
+            3: 'priority-debug',
+            4: 'priority-info',
+            5: 'priority-warn',
+            6: 'priority-error',
+            7: 'priority-assert' }
+        priority_class = ''
+        if 'priority' in entry and entry['priority'] in PRIORITY_CLASSES:
+            priority_class = PRIORITY_CLASSES[entry['priority']]
     %>
 
     ## Put a separator between entries that are separated in time.
     % if timestamp_diff_secs > 10:
-    <hr>
+        <hr>
     % endif
 
     <div class="status-entry">
         <div class="status-first-line">
-            <span class="timestamp">${timestamp_str} [+${timestamp_diff_str}s]</span>
+            <span class="timestamp">${timestamp_display(entry['timestamp'])} [+${timestamp_diff_str}s]</span>
 
-            <span class="status-entry-id">${entry['id']}</span>
+            <span class="status-entry-id ${priority_class}">${entry['id']}</span>
 
             <span class="format-args">
                 % if entry['formatArgs'] and len(entry['formatArgs']) == 1:
     </div>
 </%def>
 
+<%def name="diagnostic_history_row(entry, last_timestamp)">
+    <%
+        timestamp_diff_secs, timestamp_diff_str = get_timestamp_diff(last_timestamp, entry['timestamp'])
+    %>
+
+    ## Put a separator between entries that are separated in time.
+    % if timestamp_diff_secs > 10:
+        <hr>
+    % endif
+
+    <div class="diagnostic-entry">
+        <span class="timestamp">${timestamp_display(entry['timestamp'])} [+${timestamp_diff_str}s]</span>
+
+        <span class="diagnostic-entry-msg">${entry['msg']}</span>
+
+        ## We special-case some of the common diagnostic entries
+        % if entry['msg'] == 'ConnectingServer':
+            <span>${entry['data']['ipAddress']}</span>
+        % else:
+            <span>${repr(entry['data'])}</span>
+        % endif
+    </div>
+</%def>
+
 <h1>Status History</h1>
-<% last_timestamp = None %>
-% for entry in status_history:
-    ${status_history_row(entry, last_timestamp)}
+<%
+    last_timestamp = None
+
+    # We want the diagnostic entries to appear inline chronologically with the
+    # status entries, so we'll merge the lists and process them together.
+    status_diagnostic_history = status_history
+    if diagnostic_history:
+        status_diagnostic_history += diagnostic_history
+    status_diagnostic_history = sorted(status_diagnostic_history,
+                                       key=itemgetter('timestamp'))
+%>
+% for entry in status_diagnostic_history:
+    % if 'formatArgs' in entry:
+        ${status_history_row(entry, last_timestamp)}
+    % else:
+        ${diagnostic_history_row(entry, last_timestamp)}
+    % endif
     <% last_timestamp = entry['timestamp'] %>
 % endfor
 '''
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.