Merge branch 'master' into restcomm1657

* master: (28 commits)
  Update on DigestAuthentication unit test
  update rn wording
  updated RN wording
  updated RNs
  updated release notes
  updated migration notes
  reverted unused change
  Added extra log statements around the check of client Authorization header
  Updated External project version
  DigestAuthentication unit test
  tha patch
  corected data
  corrected test data
  dial DID with proper org
  test data
  Fixes for 8.4.0 release notes. This close RESTCOMM-2054
  Patch for race condition when SBC is enabled, where the Cancel from one of the forking branches will cause the initial call state to be marked as Canceled and thus the Bye wont be send at the end of the call. This refer to RESTCOMM-1932
  added release notes log,and link to main page
  added ReferOrganizationTest
  refactored for readability
  ...

docker-compose.yml

@@ -137,4 +137,4 @@ services:
     # Uncomment below to enable logs rotation. (Rotate using 3 files 20m each file)
     #  options:
     #    max-size: "20m"
-    #    max-file: "3"
+    #    max-file: "3"

restcomm/configuration/config-scripts/as7-config-scripts/restcomm/autoconfig.d/config-restcomm.sh

@@ -883,7 +883,14 @@ else
 fi
 
 if [ -z "$MS_ADDRESS" ]; then
-		MS_ADDRESS=$BIND_ADDRESS
+    MS_ADDRESS=$BIND_ADDRESS
+fi
+
+if [ "a" == "a$MGCP_LOCAL_ADDRESS" ]; then
+    # $MGCP_LOCAL_ADDRESS is empty, so just use bind address
+    MGCP_ADDRESS=$BIND_ADDRESS
+else
+    MGCP_ADDRESS=$MGCP_LOCAL_ADDRESS
 fi
 
 configDidProvisionManager "$DID_LOGIN" "$DID_PASSWORD" "$DID_ENDPOINT" "$DID_SITEID" "$HOSTFORDID" "$DID_ACCOUNTID" "$SMPP_SYSTEM_TYPE" "$DID_URIPORT"
@@ -892,7 +899,7 @@ configSmsAggregator "$SMS_OUTBOUND_PROXY" "$SMS_PREFIX"
 configSpeechRecognizer "$ISPEECH_KEY"
 configSpeechSynthesizers
 configTelestaxProxy "$ACTIVE_PROXY" "$TP_LOGIN" "$TP_PASSWORD" "$INSTANCE_ID" "$PROXY_IP" "$SITE_ID"
-configMediaServerManager "$BIND_ADDRESS" "$MS_ADDRESS" "$MEDIASERVER_EXTERNAL_ADDRESS"
+configMediaServerManager "$MGCP_ADDRESS" "$MS_ADDRESS" "$MEDIASERVER_EXTERNAL_ADDRESS"
 configSMPPAccount "$SMPP_ACTIVATE" "$SMPP_SYSTEM_ID" "$SMPP_PASSWORD" "$SMPP_SYSTEM_TYPE" "$SMPP_PEER_IP" "$SMPP_PEER_PORT" "$SMPP_SOURCE_MAP" "$SMPP_DEST_MAP" "$SMPP_INBOUND_ENCODING" "$SMPP_OUTBOUND_ENCODING"
 configRestCommURIs
 updateRecordingsPath

restcomm/configuration/config-scripts/as7-config-scripts/restcomm/restcomm.conf

@@ -24,6 +24,8 @@ MS_EXTERNAL=false
 MS_ADDRESS=''
 #Values rms: RestComm Media Server, xms: Dialogic Media Server
 MS_COMPATIBILITY_MODE=rms
+# The IP address that the MGCP `media-server-manager` is using to contact the Media Server.
+MGCP_LOCAL_ADDRESS=''
 LOCALMGCP=2727
 REMOTEMGCP=2427
 MGCP_RESPONSE_TIMEOUT=500
@@ -148,4 +150,4 @@ CONFERENCE_TIMEOUT="14400"
 #SDR Service configuration
 SDR_SERVICE_CLASS=''
 SDR_SERVICE_HTTP_URI=''
-SDR_SERVICE_AMQP_URI=''
+SDR_SERVICE_AMQP_URI=''

restcomm/restcomm.commons/src/main/java/org/restcomm/connect/commons/util/DigestAuthentication.java

@@ -144,4 +144,12 @@ public static String HA1(String username, String realm, String password){
         ha1 = DigestAuthentication.H(username+":"+realm+":"+password, algorithm);
         return ha1;
     }
+
+
+    //USed for unit testing
+    public static String HA1(String username, String realm, String password, String algorithm){
+        String ha1 = "";
+        ha1 = DigestAuthentication.H(username+":"+realm+":"+password, algorithm);
+        return ha1;
+    }
 }

restcomm/restcomm.commons/src/test/java/org/restcomm/connect/commons/DigestAuthenticationTest.java

@@ -0,0 +1,79 @@
+/*
+ *  TeleStax, Open Source Cloud Communications
+ *  Copyright 2011-2018, Telestax Inc and individual contributors
+ *  by the @authors tag.
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  under the terms of the GNU Affero General Public License as
+ *  published by the Free Software Foundation; either version 3 of
+ *  the License, or (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU Affero General Public License for more details.
+ *
+ *  You should have received a copy of the GNU Affero General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>
+ */
+
+package org.restcomm.connect.commons;
+
+import org.junit.Test;
+import org.restcomm.connect.commons.dao.Sid;
+import org.restcomm.connect.commons.util.DigestAuthentication;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class DigestAuthenticationTest {
+
+    private String client = "alice0000000";
+    private String domain = "org0000000.restcomm.com";
+    private String password = "1234";
+    private String proxyAuthHeader = "Digest username=\"alice0000000\",realm=\"org0000000.restcomm.com\",cnonce=\"6b8b4567\",nc=00000001,qop=auth,uri=\"sip:172.31.45.30:5080\",nonce=\"61343361383534392d633237372d343\",response=\"bc322276e42a123c53c2ed6f53d5e7c7\",algorithm=MD5";
+
+    @Test
+    public void testAuth(){
+        String hashedPass = DigestAuthentication.HA1(client, domain, password, "MD5");
+
+        assertEquals("9b11a2924d0881aca84f9db97f834d99", hashedPass);
+
+        assertTrue(permitted(proxyAuthHeader, "INVITE", hashedPass));
+
+    }
+
+    static boolean permitted(final String authorization, final String method, String clientPassword) {
+        final Map<String, String> map = authHeaderToMap(authorization);
+        String user = map.get("username");
+        final String algorithm = map.get("algorithm");
+        final String realm = map.get("realm");
+        final String uri = map.get("uri");
+        final String nonce = map.get("nonce");
+        final String nc = map.get("nc");
+        final String cnonce = map.get("cnonce");
+        final String qop = map.get("qop");
+        final String response = map.get("response");
+        final String password2 = clientPassword;
+        final String result = DigestAuthentication.response(algorithm, user, realm, "", password2, nonce, nc, cnonce,
+                method, uri, null, qop);
+        return result.equals(response);
+    }
+
+    private static Map<String, String> authHeaderToMap(final String header) {
+        final Map<String, String> map = new HashMap<String, String>();
+        final int endOfScheme = header.indexOf(" ");
+        map.put("scheme", header.substring(0, endOfScheme).trim());
+        final String[] tokens = header.substring(endOfScheme + 1).split(",");
+        for (final String token : tokens) {
+            final String[] values = token.trim().split("=",2); //Issue #935, split only for first occurrence of "="
+            map.put(values[0].toLowerCase(), values[1].replace("\"", ""));
+        }
+
+        return map;
+    }
+
+}

restcomm/restcomm.docs/sources-asciidoc/src/main/asciidoc/index.adoc

@@ -21,3 +21,6 @@ There is multiple ways to interact with Restcomm Connect to build applications
 * The <<tutorials/index.adoc#tutorials,Restcomm Tutorials>> will help you in creating sample applications that cover common use cases in a variety of languages using the Restcomm API, RCML or Visual Designer. Download, test, and tweak for yourself. 
 
 * The <<configuration/index.adoc#Configuration,Configuration>> will help you in setting up Restcomm Connect on your own server or development environment to build powerful apps locally.
+
+
+Find <<release-notes.adoc#Release Notes,Release Notes>> here

restcomm/restcomm.docs/sources-asciidoc/src/main/asciidoc/release-notes.adoc

@@ -0,0 +1,58 @@
+= Restcomm-Connect Release Notes
+
+These notes group both Community and Telscale changes in a single document.
+
+=== Tags
+
+The folowing tags are used to categorize and state the scope of a change
+
+* **security improvement** tags changes related to security
+* **commercial** tags changes that are available only in the commercial RestcommOne product
+
+
+//add release-notes with newer on top
+== 8.4.0 version
+=== New Features
+// New features (whether major or minor) go here
+* Clients passwords are now hashed in DB - **security improvement**
+* Profiles allow arbitrary properties to be saved/retrieved.
+* Added configurable SBC mode which if enabled will disable all NAT handling operations
+
+=== Breaking Changes
+// draws attention to functionality that is getting removed
+* Accessing Olympus WebRTC from Console now requires to login again. This because of the new feature to hash passwords
+* Removed default clients `alice` and `bob`
+
+=== Bug Fixes
+// any difference in functionality
+* Dial Timeout does not cancel task when Callee is busy
+* Configurable inbound/outbound SMPP encoding
+* Race condition on sending BYE to incoming call for a dial fork scenario
+* Fixed SDR event for SMS - **commercial**
+* Fixed REFER (Call Transfer) support to work with organizations
+
+
+=== Migration Notes
+// Things to consider during migration from previous release
+* Clients password are considered to be MD5 hashed in DB. Database migration
+scripts are available in **commercial** version. Database migration script will ensure existing clients passwords are properly migrated.
+** Database migration script will automatically take a backup of complete database before making any changes
+** Please take a backup of restcomm clients table (to be used in case we need to rollback as explained below)
+** In case of rollback to older version, kindly restore restcomm clients table only.
+
+=== External Dependencies Updates
+// any dependencies
+* Console updated to version **__8.4.0__**
+** Integrated Feature Access Control (FAC) limitations
+** Updated Console Look & Feel to match new Restcomm branding
+** Implemented new Sign In page in Console
+** Use Designer location in Console from configuration file
+* Designer updated to version **__1.2.0-131__**
+** Integrated Feature Access Control (FAC) limitations
+** Improved Designer performance with better xstream usage
+** Designer Look & Feel to match new Restcomm branding
+* Olympus WebRTC Demo updated to version **__1.1.0-167__**
+** Fixed an issue in WebRTC Demo jain-sip library, parsing some headers
+** Improve WebRTC Demo UX by showing incoming call screen on top even if caller is not the selected contact
+** Add additional configuration to WebRTC Demo for specifying client-specific parameters
+** Improve WebRTC Xirsys integration by checking for actual success response and using domain property as namespace (now required)

restcomm/restcomm.interpreter/src/main/java/org/restcomm/connect/interpreter/SIPOrganizationUtil.java

@@ -19,6 +19,8 @@
  */
 package org.restcomm.connect.interpreter;
 
+import javax.servlet.sip.Address;
+import javax.servlet.sip.ServletParseException;
 import javax.servlet.sip.SipServletRequest;
 import javax.servlet.sip.SipURI;
 import org.apache.log4j.Logger;
@@ -50,6 +52,18 @@ public static Sid searchOrganizationBySIPRequest(OrganizationsDao orgDao, SipSer
             // try to get destinationOrganizationSid from toUri
             destinationOrganizationSid = getOrganizationSidBySipURIHost(orgDao, (SipURI) request.getTo().getURI());
         }
+        if (destinationOrganizationSid == null) {
+            // try to get destinationOrganizationSid from Refer-To
+            Address referAddress;
+            try {
+                referAddress = request.getAddressHeader("Refer-To");
+                if(referAddress != null){
+                    destinationOrganizationSid = getOrganizationSidBySipURIHost(orgDao, (SipURI) referAddress.getURI());
+                }
+            } catch (ServletParseException e) {
+                logger.error(e);
+            }
+        }
         return destinationOrganizationSid;
     }
 }

restcomm/restcomm.interpreter/src/main/java/org/restcomm/connect/interpreter/VoiceInterpreter.java

@@ -1244,7 +1244,7 @@ private void onCallStateChanged(Object message, ActorRef sender) throws Transiti
             case RINGING:
                 break;
             case CANCELED:
-                if (is(initializingBridge) || is(acquiringOutboundCallInfo) || is(bridging) || is(bridged)) {
+                if (is(creatingBridge) || is(initializingBridge) || is(acquiringOutboundCallInfo) || is(bridging) || is(bridged)) {
                     //This is a canceled branch from a previous forking call. We need to destroy the branch
 //                    removeDialBranch(message, sender);
                     callManager.tell(new DestroyCall(sender), self());

restcomm/restcomm.telephony.api/src/main/java/org/restcomm/connect/telephony/api/util/CallControlHelper.java

@@ -31,6 +31,7 @@
 import javax.servlet.sip.SipServletResponse;
 import javax.servlet.sip.SipURI;
 
+import org.apache.log4j.Logger;
 import org.restcomm.connect.dao.ClientsDao;
 import org.restcomm.connect.dao.DaoManager;
 import org.restcomm.connect.dao.entities.Client;
@@ -47,6 +48,7 @@
  *
  */
 public class CallControlHelper {
+    private static Logger logger = Logger.getLogger(CallControlHelper.class);
 
     static boolean permitted(final String authorization, final String method, DaoManager daoManager, final Sid organizationSid) {
         final Map<String, String> map = authHeaderToMap(authorization);
@@ -66,8 +68,16 @@ static boolean permitted(final String authorization, final String method, DaoMan
             final String password2 = client.getPassword();
             final String result = DigestAuthentication.response(algorithm, user, realm, "", password2, nonce, nc, cnonce,
                     method, uri, null, qop);
+            if (logger.isDebugEnabled()) {
+                String msg = String.format("Provided response [%s], generated response [%s]", response, result);
+                logger.debug(msg);
+            }
             return result.equals(response);
         } else {
+            if (logger.isDebugEnabled()) {
+                String msg = String.format("Authorization check failed, [if(client==null) evaluates %s] or [if(client.getStatus()==Client.ENABLED) evaluates %s]", client!=null, Client.ENABLED == client.getStatus());
+                logger.debug(msg);
+            }
             return false;
         }
     }
@@ -85,6 +95,10 @@ public static boolean checkAuthentication(SipServletRequest request, DaoManager
         final String authorization = request.getHeader("Proxy-Authorization");
         final String method = request.getMethod();
         if (authorization == null || !CallControlHelper.permitted(authorization, method, storage, organizationSid)) {
+            if (logger.isDebugEnabled()) {
+                String msg = String.format("Either authorization header is null [if(authorization==null) evaluates %s], or CallControlHelper.permitted() method failed, will send \"407 Proxy Authentication required\"", authorization==null);
+                logger.debug(msg);
+            }
             authenticate(request, storage.getOrganizationsDao().getOrganization(organizationSid).getDomainName());
             return false;
         } else {

restcomm/restcomm.testsuite/src/test/java/org/restcomm/connect/testsuite/UssdPullTest.java

@@ -137,6 +137,7 @@ public void after() throws Exception {
     }
 
     @Test
+    @Category(UnstableTests.class)
     public void testUssdPull() {
         final SipCall bobCall = bobPhone.createSipCall();
         bobCall.initiateOutgoingCall(bobContact, ussdPullDid, null, UssdPullTestMessages.ussdClientRequestBody, "application", "vnd.3gpp.ussd+xml", null, null);